Add example selenium script for the atom hackathon. - brcon2023-hackathons - Bitreichcon 2023 Hackathon Repository
 (HTM) git clone git://bitreich.org/brcon2023-hackathons git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/brcon2023-hackathons
 (DIR) Log
 (DIR) Files
 (DIR) Refs
 (DIR) Tags
       ---
 (DIR) commit a7cd0c547c792f74b7784cc0a8c806380a28ca2f
 (DIR) parent 2922c09dc4919dcea4ac331bbaa4e373ba4ccc4a
 (HTM) Author: Christoph Lohmann <20h@r-36.net>
       Date:   Thu, 10 Aug 2023 16:10:01 +0200
       
       Add example selenium script for the atom hackathon.
       
       Diffstat:
         A sfeed-atom/kvssachsen2atom          |     121 +++++++++++++++++++++++++++++++
       
       1 file changed, 121 insertions(+), 0 deletions(-)
       ---
 (DIR) diff --git a/sfeed-atom/kvssachsen2atom b/sfeed-atom/kvssachsen2atom
       @@ -0,0 +1,121 @@
       +#!/usr/bin/env python
       +# coding=utf-8
       +#
       +# Copy me if you can.
       +# by 20h
       +#
       +
       +import os
       +import sys
       +import getopt
       +
       +from selenium import webdriver
       +from selenium.webdriver.chrome.options import Options as chromeoptions
       +from selenium.webdriver.support.ui import WebDriverWait
       +from selenium.webdriver.support import expected_conditions as EC
       +from selenium.webdriver.common.by import By
       +
       +from datetime import datetime
       +import pytz
       +
       +def usage(app):
       +        app = os.path.basename(app)
       +        sys.stderr.write("usage: %s [-h] URI\n" % (app))
       +        sys.exit(1)
       +
       +def main(args):
       +        try:
       +                opts, largs = getopt.getopt(args[1:], "h")
       +        except getopt.GetoptError as err:
       +                print(str(err))
       +                usage(args[0])
       +        
       +        for o, a in opts:
       +                if o == "-h":
       +                        usage(args[0])
       +                else:
       +                        assert False, "unhandled option"
       +
       +        if len(largs) < 1:
       +                usage(args[0])
       +
       +        link = largs[0]
       +
       +        options = chromeoptions()
       +        chromearguments = [
       +                "headless",
       +                "no-sandbox",
       +                "disable-extensions",
       +                "disable-dev-shm-usage",
       +                "start-maximized",
       +                "window-size=1900,1080",
       +                "disable-gpu"
       +        ]
       +        for carg in chromearguments:
       +                options.add_argument(carg)
       +
       +        driver = webdriver.Chrome(options=options)
       +        driver.get(link)
       +
       +        isnews = WebDriverWait(driver=driver, timeout=60).until(
       +                        EC.presence_of_element_located((By.XPATH,
       +                                "//div[@data-last-letter]")
       +                        )
       +        )
       +        newslist = driver.find_elements(By.XPATH, "//div[@data-filter-target=\"list\"]")[0]
       +
       +        title = driver.find_elements(By.XPATH, "//meta[@property=\"og:title\"]")[0].get_attribute("content")
       +        description = title
       +        globaltags = ""
       +
       +        print("""<?xml version="1.0" encoding="utf-8"?>""")
       +        print("""<feed xmlns="http://www.w3.org/2005/Atom">""")
       +        print("\t<title><![CDATA[%s]]></title>" % (title))
       +        print("\t<subtitle><![CDATA[%s]]></subtitle>" % (description))
       +        print("\t<id>%s</id>" % (link))
       +        print("\t<link href=\"%s\" rel=\"self\" />" % (link))
       +        print("\t<link href=\"%s\" />" % (link))
       +
       +        utcnow = datetime.now(pytz.utc)
       +        print("\t<updated>%s</updated>" % (utcnow.isoformat()))
       +
       +        articles = newslist.find_elements(By.XPATH, "./div")
       +        baselink = "/".join(link.split("/", 3)[:-1])
       +        for article in articles[::-1]:
       +                link = article.find_elements(By.XPATH, "./a")[0]
       +                plink = link.get_attribute("href")
       +                if not plink.startswith("http"):
       +                        plink = "%s/%s" % (baselink, plink)
       +                ptitle = link.get_attribute("data-title")
       +                pcontent = article.text
       +                pauthor = "sachsen@kvsachsen.de"
       +
       +                # Normalize datetime.
       +                updateds = article.find_elements(By.XPATH, ".//time")[0].text
       +                try:
       +                        dtupdated = datetime.strptime(updateds, "%d.%m.%Y")
       +                except ValueError:
       +                        continue
       +
       +                dtupdated = dtupdated.replace(hour=12, minute=0,\
       +                                second=0, tzinfo=pytz.utc)
       +                if dtupdated.year > utcnow.year:
       +                        dtupdated = dtupdated.replace(year=utcnow.year)
       +                pupdated = dtupdated
       +
       +                print("\t<entry>")
       +                print("\t\t<id>%s</id>" % (plink))
       +                print("\t\t<title><![CDATA[%s]]></title>" % (ptitle))
       +                print("\t\t<link href=\"%s\" />" % (plink))
       +                print("\t\t<author><name>%s</name></author>" % (pauthor))
       +                print("\t\t<updated>%s</updated>" % (pupdated.isoformat()))
       +                print("\t\t<content><![CDATA[%s]]></content>" % (pcontent))
       +                print("\t</entry>")
       +        
       +        print("</feed>")
       +
       +        return 0
       +
       +if __name__ == "__main__":
       +        sys.exit(main(sys.argv))
       +