archive selenium script - randomcrap - random crap programs of varying quality
 (HTM) git clone git://git.codemadness.org/randomcrap
 (DIR) Log
 (DIR) Files
 (DIR) Refs
 (DIR) README
 (DIR) LICENSE
       ---
 (DIR) commit 509d67a4275fb11eee7ba96490e6138da9c8f4a9
 (DIR) parent 0332d1d50a2f4fdf7249747ac7c49644ed5f22a9
 (HTM) Author: Hiltjo Posthuma <hiltjo@codemadness.org>
       Date:   Tue, 28 Nov 2023 18:27:02 +0100
       
       archive selenium script
       
       Diffstat:
         A selenium/selenium_crawl_tsv.py      |     118 +++++++++++++++++++++++++++++++
       
       1 file changed, 118 insertions(+), 0 deletions(-)
       ---
 (DIR) diff --git a/selenium/selenium_crawl_tsv.py b/selenium/selenium_crawl_tsv.py
       @@ -0,0 +1,118 @@
       +from selenium import webdriver
       +from selenium.webdriver.common.by import By
       +
       +from selenium.webdriver.firefox.options import Options
       +from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
       +
       +import sys
       +from datetime import datetime
       +
       +def make_escape_content_trans():
       +    m = {}
       +    for i in range(0, 32):
       +        m[i] = ""
       +    m[0x7f] = "" # DEL
       +    # replace
       +    m["\\"] = "\\\\"
       +    m["\n"] = "\\n"
       +    m["\t"] = "\\t"
       +
       +    return str.maketrans(m)
       +
       +def make_escape_field_trans():
       +    m = {}
       +    for i in range(0, 32):
       +        m[i] = ""
       +    m[0x7f] = "" # DEL
       +    # replace
       +    m["\n"] = " "
       +    m["\t"] = " "
       +
       +    return str.maketrans(m)
       +
       +escape_content_tbl = make_escape_content_trans()
       +escape_field_tbl = make_escape_field_trans()
       +
       +def escape_content(s):
       +    return s.translate(escape_content_tbl).strip()
       +
       +def escape_field(s):
       +    return s.translate(escape_field_tbl).strip()
       +
       +if len(sys.argv) > 1:
       +    url = sys.argv[1]
       +else:
       +    print("usage: <url>")
       +    sys.exit(1)
       +
       +options = Options()
       +options.add_argument("--headless")
       +
       +# use existing profile:
       +
       +#options.add_argument("--profile")
       +#profile_path = "/home/hiltjo/.mozilla/firefox/something.default-release"
       +# NOTE: must not be running at the same time.
       +#options.add_argument(profile_path)
       +#options.set_preference("profile", profile_path)
       +
       +# setup custom profile:
       +# JS disabled
       +options.set_preference("javascript.enabled", False)
       +# disable stylesheet
       +options.set_preference("permissions.default.stylesheet", 2)
       +# disable image loading
       +options.set_preference("permissions.default.image", 2)
       +# override user-agent.
       +#options.set_preference("general.useragent.override", "whatever you want")
       +
       +driver = webdriver.Firefox(options=options)
       +
       +# set timeouts
       +#driver.implicitly_wait(10)
       +
       +# get the page
       +driver.get(url)
       +
       +# print page title
       +#print(driver.title)
       +
       +#pagesource = driver.execute_script("return document.body.InnerHTML;")
       +#print(pagesource)
       +#print(driver.page_source)
       +#outer_html = driver.find_element(By.XPATH, "//body").get_attribute("outerHTML")
       +
       +#outer_html = driver.find_element(By.TAG_NAME, "html").get_attribute("outerHTML")
       +#print(outer_html)
       +
       +# show all links on a page
       +#links = driver.find_elements(By.TAG_NAME, "a")
       +anchors = driver.find_elements(By.CSS_SELECTOR, "main a")
       +links = []
       +for anchor in anchors:
       +    href = anchor.get_attribute("href")
       +    text = anchor.text
       +    if len(href):
       +       links.append({"href": href, "text": text})
       +
       +for link in links:
       +    driver.get(link["href"])
       +
       +    # parse timestamp.
       +    time = driver.find_element(By.TAG_NAME, "time")
       +    ts = datetime.strptime(time.text, "%Y-%m-%d")
       +    ts = int(ts.timestamp())
       +
       +    content = driver.find_element(By.CSS_SELECTOR, "article").get_attribute("outerHTML")
       +    title = driver.title
       +    title = title.replace(" - Codemadness", "")
       +
       +    # escape fields
       +    content = escape_content(content)
       +    title = escape_field(title)
       +    link = escape_field(link["href"])
       +
       +    print("%d\t%s\t%s\t%s\thtml" % (ts, title, link, content))
       +
       +driver.close()
       +driver.quit()