#!/usr/bin/python3

# TODO: https://www.bergfex.de/st-andreasberg-matthias-schmidt-berg/webcams/c5771/
#       https://www.wetter.com/hd-live-webcams/deutschland/st-andreasberg-sommerrodelbahn/605922f525ffa/

import re
import requests
from bs4 import BeautifulSoup
import sys
from requests.structures import CaseInsensitiveDict
import datetime
import os
from PIL import Image
from io import BytesIO
import time
import random

# https://stackoverflow.com/questions/27981545/suppress-insecurerequestwarning-unverified-https-request-is-being-made-in-pytho
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

debug=1 # set >1 for even more details
use_iprotator=False
SLEEPTIMEPROJECTS=60

localbasedir='/var/lib/brocken'

fixed_height = 1080

# view-source:https://webtv.feratel.com/webtv/?cam=3177&t=12&design=v4
site = 'https://webtv.feratel.com/'
sitepage = site+'webtv/' # last slash is important!


params = dict()
#params['cam']="3177" # will be overridden in param loop to fetch all image sources
params['t']="12"
params['design']="v4"

#https://reqbin.com/req/python/gplpbyk6/get-request-like-mozilla-firefox
headers = CaseInsensitiveDict()
# headers["Connection"] = "keep-alive"
# # headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0"
# # headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0"
# headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0"
# headers["Upgrade-Insecure-Requests"] = "1"
# headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
# headers["Accept-Language"] = "en-US,en;q=0.5"
# headers["Accept-Encoding"] = "gzip, deflate"

# https://reqbin.com/req/python/mur9czhj/get-request-like-google-chrome
headers["Connection"] = "keep-alive"
headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"
headers["Upgrade-Insecure-Requests"] = "1"
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
headers["Accept-Language"] = "en-US,en;q=0.9"
headers["Accept-Encoding"] = "gzip, deflate"

if use_iprotator:
    # https://stackoverflow.com/questions/55872164/how-to-rotate-proxies-on-a-python-requests/68451842#68451842
    from requests_ip_rotator import ApiGateway
    gateway = ApiGateway(site, regions=["eu-central-1"])
    gateway.start()
    session = requests.Session()
    session.mount(site, gateway)

today = datetime.date.today()
imgs = dict()
imgdate = None

if debug > 0:
    import pprint
for param in [ "3175", "3176", "3177", "23175", "3257" ]:
    params['cam']=param
    if use_iprotator:
        # to use Gateway:
        # response = session.get(sitepage, params=params, headers = headers)
        link=sitepage+'?cam=%(cam)s&t=%(t)s&design=%(design)s' % params
        if debug > 1:
            print(link)
        response = session.get(link, headers = headers)
        if debug > 1:
            pprint.pprint(vars(response))
            pprint.pprint(vars(response.connection))
            pprint.pprint(vars(response.raw))
    else:
        if debug >1:
            print(params)
        try:
            response = requests.get(sitepage, params=params, headers = headers, verify=False, timeout=10)
        except:
            print("Unable to fetch sitepage %s - continue with other images" % sitepage)
            continue
        if debug >1:
            pprint.pprint(vars(response))
    if response.status_code == 404:
        print("Failed to access %s (status=%i). Exit." % (sitepage, response.status_code))
        if use_iprotator:
            print("Ip-rotator failed to download ", link)
            gateway.shutdown() 
        exit(1)
    if response.status_code != 200:
        print("Non-default response status code: ", response.status_code)
    soup = BeautifulSoup(response.text, "lxml")
    imglinks = soup.find_all("div", attrs={"class":"col-sm-5 tophotel_item_pic"})
    if debug > 1:
        print("DEBUG: param=%s, imglinks = '%s'" % (param, imglinks))
    if imglinks:
        for iml in imglinks:
            im = iml.find("img")
            if debug > 1:
                print("DEBUG: ", im)
            if im["alt"] == "":
                alt=re.sub(r'^([^,]+).*', '\\1', im["data-title"].replace("\n",""))
            else:
                alt=im["alt"]
            if debug > 1:
                print("Title = %s, Alt = %s, Id = %s" % (im["data-title"], alt, im["id"]) )
            id=""
            if alt == "Braunlage - Wurmberg Alm":
                id = re.sub(r'^ *snaps', '_', im["id"])
            tod = "%04i-%02i-%02i " % (today.year,today.month,today.day) + re.sub(r'^.*(\d\d:\d\d [AP]M).*', '\\1', im["data-title"].replace("\n",""))
            imgdate = datetime.datetime.strptime( tod, "%Y-%m-%d %I:%M %p").strftime("%Y%m%d%H%M%S")
            im["src"] = re.sub('&amp;', '&', im["src"])
            imgs[re.sub(r' *\(.+\)', '', alt).replace(" ", "_").replace("ä", "ae").replace("ü", "ue")+id] = [re.sub(r'\?xdesign=v.*', '', im["src"]), imgdate]
    else:
        iml = soup.find("div", attrs={"id":"livebildDIV"})
        title = soup.find("img", attrs={"class":"cams_img_a"})
        try:
            imgs[re.sub(r'^.* - (.+), <.*$', '\\1', title["alt"]).replace(" ", "_").replace("ö", "oe")] = [re.sub(r'^.*(https:.*)\?xdesign.*', '\\1', iml["style"]), imgdate]
        except TypeError:
            if not iml:
                print("Die haben mich scheinbar ausgesperrt :-(")
                if use_iprotator:
                    gateway.shutdown() 
                exit(1)
    if debug == 0:
        time.sleep(SLEEPTIMEPROJECTS + random.randint(-30,30))

# imglinks = soup.find("div") # , attrs={"class":"col-sm-5 tophotel_item_pic"})

for i in imgs.keys():
    url = imgs[i][0]
    if debug > 1:
        print("Next URL: ", url)
    remotefilename = re.search(r'/([\w_-]+[.]jpeg)$', url)
    if debug > 1:
         print("Try downloading remotefilename ", remotefilename)
    if not remotefilename:
         if debug > 0:
             print("Regex didn't match with the url: {}".format(url))
         continue
    dirname = os.path.join(localbasedir, i)
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    imgfile = "%s_%s_%s" % (imgs[i][1], i, remotefilename.group(1))
    if debug > 1:
        print(imgfile)
    filename = os.path.join(dirname, imgfile)
    hasstored = False
    for root, directories, file in os.walk(dirname):
        for file in file:
            if(file.endswith(".jpeg")):
                stored = re.search(remotefilename.group(1), file)
                if stored:
                    if debug > 1:
                        print("File %s was previously stored" % filename)
                    hasstored = True
                    break
    if debug > 1:
        print("%s was stored before" % filename)
    if hasstored:
        if debug > 1:
            print("continuing")
        continue
#    with open(filename, 'wb') as f: # PIL has a better compression and enables resizing
    if debug > 1:
        print("Try downloading")
    if 'http' not in url:
        # sometimes an image source can be relative
        # if it is provide the base url which also happens
        # to be the site variable atm.
        url = '{}{}'.format(site, url)
    if debug > 1:
        print("get URL ", url)
    try:
        response = requests.get(url, timeout=10)
    except TimeoutException:
        print("Unable to get url %s - continue with other images" % url)
        continue
    finally:
        if debug > 1:
            print("Finally ", url)
    image = Image.open(BytesIO(response.content))
    if image.size[1] > 1080:
        height_percent = (fixed_height / float(image.size[1]))
        width_size = int((float(image.size[0]) * float(height_percent)))
        image = image.resize((width_size, fixed_height)) # , PIL.Image.NEAREST
        if debug > 1:
            print("resizing %s to %i, %i" % (filename, width_size, fixed_height))
    image.save(filename)
    if debug > 0:
        print("Feratel saved: ", filename)
#        f.write(response.content)
    currentfile = os.path.join(dirname, "%s_current.jpeg" % (i))
    if os.path.exists(currentfile):
        os.unlink(currentfile)
    os.symlink(imgfile, currentfile)
    ## This failed strangely in some cases with non-zero return value
    # os.system("ln -sf %s %s" % (imgfile, currentfile))
    ## For debugging the following was used - but the code above should do the trick as well
    #lncmd = "ln -sf %s %s" % (imgfile, currentfile)
    #with open('debug.log', 'a') as dbg:
    #    if os.system(lncmd):
    #        dbg.write("# There was an issue with: \n" + lncmd + "\n")
    #    else:
    #        dbg.write("# Success: " + lncmd + "\n")

if use_iprotator:
    gateway.shutdown() 
