more refactoring and cleanup - linuxgaming - Linux gaming aggregate tool, built to test out NodeJS.
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
---
(DIR) commit 5917aef9ef70f99064dd0db3711c4dff95a73db4
(DIR) parent f1830964e2f2a24fab826cbdc8bac5ce3f3548a7
(HTM) Author: Jay Scott <me@jay.scot>
Date: Wed, 18 Jul 2018 19:00:19 +0100
more refactoring and cleanup
Diffstat:
M TODO.md | 8 --------
M linuxgaming/__init__.py | 20 ++++++++++----------
M linuxgaming/database.py | 21 +++++++++++++++------
M linuxgaming/details.py | 34 ++++++++-----------------------
M linuxgaming/search.py | 23 ++++++++---------------
M linuxgaming/update.py | 169 ++++++++++++-------------------
A linuxgaming/util.py | 13 +++++++++++++
7 files changed, 118 insertions(+), 170 deletions(-)
---
(DIR) diff --git a/TODO.md b/TODO.md
@@ -1,14 +1,6 @@
# TODO
-## pre 1.0
-
-
- - Refactor code
-
-
-## other
-
- Add menu with source info / about etc
- Move updates to AWS Lambda function
- Add API
(DIR) diff --git a/linuxgaming/__init__.py b/linuxgaming/__init__.py
@@ -35,30 +35,30 @@ def create_app():
@app.route("/")
def home():
- result = database.db_search(
- {"date": {'$gte': datetime.now() - timedelta(hours=24)}})
+ result = database.find_all({
+ "date": {
+ '$gte': datetime.now() - timedelta(hours=24)
+ }
+ })
return render_template('pages/home.html', entries=result)
@app.errorhandler(500)
def internal_error(error):
app.logger.error('internal error %s', error)
return render_template(
- "message.html",
- icon="frown",
- msg="Something went wrong!"), 500
+ "message.html", icon="frown", msg="Something went wrong!"), 500
@app.errorhandler(404)
- def page_not_found(e):
+ def page_not_found():
app.logger.info('page not found')
return render_template(
- "message.html",
- icon="frown",
- msg="I think you are lost!"), 404
+ "message.html", icon="frown", msg="I think you are lost!"), 404
@app.template_filter('strftime')
- def _jinja2_filter_datetime(date, fmt=None):
+ def _jinja2_filter_datetime(date):
date = dateutil.parser.parse(str(date))
native = date.replace(tzinfo=None)
format = '%a %d %b %X %Y'
return native.strftime(format)
+
return app
(DIR) diff --git a/linuxgaming/database.py b/linuxgaming/database.py
@@ -1,14 +1,23 @@
from flask import current_app
-def db_search(query={}):
+def find_all(query={}):
try:
- d = current_app.mongo.db.items.find(
- query
- ).sort('date', -1)
- except pymongo.errors.OperationFailure:
- print("DB Error")
+ d = current_app.mongo.db.items.find(query).sort('date', -1)
+ except Exception as e:
+ current_app.logger.error('DB replace error %s', e)
return False
return d
+
+
+def replace_one(query, data, upsert=True):
+
+ try:
+ current_app.mongo.db.items.replace_one(query, data, upsert)
+ except Exception as e:
+ current_app.logger.error('DB replace error %s', e)
+ return False
+
+ return True
(DIR) diff --git a/linuxgaming/details.py b/linuxgaming/details.py
@@ -1,30 +1,15 @@
-from flask import (
- Blueprint,
- flash,
- redirect,
- render_template,
- url_for,
- current_app)
-import yaml
+from flask import (Blueprint, flash, redirect, render_template, url_for,
+ current_app)
+from . import database
+from . import util
bp = Blueprint('details', __name__, url_prefix='/details')
-def load():
- """Return the YAML parsed config file."""
- try:
- with open('config/feed_config.yaml', 'r') as ymlfile:
- cfg = yaml.load(ymlfile)
- except yaml.YAMLError as exc:
- current_app.logger.error('YAML read error %s', exc)
-
- return cfg
-
-
-@bp.route("/<path:path>", methods=('GET', 'POST'))
+@bp.route("/<path:path>", methods=["GET"])
def details(path):
"""Source details page"""
- feed_config = load()
+ feed_config = util.load_yaml()
if path in feed_config:
source_data = feed_config[path]
@@ -33,10 +18,7 @@ def details(path):
current_app.logger.info('Manual details probe %s', path)
return redirect(url_for('home'))
- source_items = current_app.mongo.db.items.find(
- {"name": path}).sort('date', -1)
+ source_items = database.find_all({"name": path})
return render_template(
- 'pages/details.html',
- data=source_data,
- items=source_items)
+ 'pages/details.html', data=source_data, items=source_items)
(DIR) diff --git a/linuxgaming/search.py b/linuxgaming/search.py
@@ -1,28 +1,21 @@
-from flask import (
- Blueprint,
- flash,
- redirect,
- url_for,
- render_template,
- current_app)
+from flask import (Blueprint, flash, redirect, url_for, render_template,
+ current_app)
+from . import database
bp = Blueprint('search', __name__, url_prefix='/search')
-@bp.route("/<path:path>", methods=('GET', 'POST'))
-def test(path):
+@bp.route("/<path:path>", methods=["GET"])
+def search(path):
pages = ['game', 'twitch', 'youtube', 'article', 'podcast']
if any(x in path for x in pages):
- result = current_app.mongo.db.items.find(
- {"type": path}).sort('date', -1)
+ result = database.find_all({"type": path})
return render_template(
- 'pages/search.html',
- entries=result,
- count=result.count())
+ 'pages/search.html', entries=result, count=result.count())
else:
flash('1337 Hacks in progress...')
current_app.logger.info('Manual search probe %s', path)
-
+
return redirect(url_for('home'))
(DIR) diff --git a/linuxgaming/update.py b/linuxgaming/update.py
@@ -1,29 +1,24 @@
from googleapiclient.discovery import build
from twitch import TwitchClient
from flask import Blueprint, render_template, current_app
-import yaml
+from . import database
+import json
+import requests
import feedparser
import dateutil.parser
+from . import util
bp = Blueprint('update', __name__, url_prefix='/update')
-def load():
-
- with open('config/feed_config.yaml', 'r') as ymlfile:
- cfg = yaml.load(ymlfile)
-
- return cfg
-
-
def parse(url):
return feedparser.parse(url).entries
-@bp.route('/rss', methods=('GET', 'POST'))
+@bp.route('/rss', methods=["GET"])
def rss_update():
- feed_config = load()
+ feed_config = util.load_yaml()
for section in feed_config:
if 'rss' not in feed_config[section]:
@@ -40,52 +35,45 @@ def rss_update():
description = ""
else:
description = feed.description
- data = {"name": section,
- "icon": feed_config[section]['icon'],
- "title": trimtitle,
- "description": description,
- "url": feed.link,
- "type": feed_config[section]['rss']['type'],
- "date": dateutil.parser.parse(feed.updated)}
-
- try:
- current_app.mongo.db.items.replace_one(
- {'title': trimtitle}, data, True)
- except Exception as e:
- current_app.logger.error('DB replace error %s', e)
- return render_template(
- "message.html", icon="frown", msg=str(e))
- return render_template(
- "message.html",
- icon="smile",
- msg="RSS feeds updated!")
+ data = {
+ "name": section,
+ "icon": feed_config[section]['icon'],
+ "title": trimtitle,
+ "description": description,
+ "url": feed.link,
+ "type": feed_config[section]['rss']['type'],
+ "date": dateutil.parser.parse(feed.updated)
+ }
+
+ database.replace_one({'title': trimtitle}, data)
+
+ return render_template("message.html", msg="RSS feeds updated!")
-@bp.route('/twitch', methods=('GET', 'POST'))
+@bp.route('/twitch', methods=["GET"])
def twitch_update():
- feed_config = load()
+ feed_config = util.load_yaml()
for section in feed_config:
if 'twitch' not in feed_config[section]:
continue
current_app.logger.info('[TWITCH] Updating %s', section)
- twitch_channelID = feed_config[section]['twitch']['twitch_id']
+ twitch_channelid = feed_config[section]['twitch']['twitch_id']
client = TwitchClient(
client_id=current_app.config['TWITCH_CLIENTID'],
- oauth_token=current_app.config['TWITCH_TOKEN']
- )
+ oauth_token=current_app.config['TWITCH_TOKEN'])
videos = client.channels.get_videos(
- twitch_channelID, # Channel ID
+ twitch_channelid, # Channel ID
10, # Limit
0, # Offset
'archive', # Broadcast type
- 'en', # Lang
- 'time', # Sort
+ 'en', # Lang
+ 'time', # Sort
)
for search_results in videos:
@@ -100,81 +88,64 @@ def twitch_update():
"date": dateutil.parser.parse(search_results['recorded_at'])
}
- try:
- current_app.mongo.db.items.replace_one(
- {'url': search_results['url']}, data, True)
- except Exception as e:
- current_app.logger.error('DB replace error %s', e)
- return render_template(
- "message.html", icon="frown", msg=str(e))
+ database.replace_one({'url': search_results['url']}, data)
- return render_template(
- "message.html",
- icon="smile",
- msg="Twitch API updated!")
+ return render_template("message.html", msg="Twitch API updated!")
-@bp.route('/youtube', methods=('GET', 'POST'))
+@bp.route('/youtube', methods=["GET"])
def youtube_update():
- DEVELOPER_KEY = current_app.config['YOUTUBE_APIKEY']
- YOUTUBE_API_SERVICE_NAME = 'youtube'
- YOUTUBE_API_VERSION = 'v3'
+ key = current_app.config['YOUTUBE_APIKEY']
+ youtube_api = 'youtube'
+ api_version = 'v3'
- feed_config = load()
+ feed_config = util.load_yaml()
for section in feed_config:
if 'youtube' not in feed_config[section]:
continue
- youtube_channelID = feed_config[section]['youtube']['channel_id']
+ youtube_channel_id = feed_config[section]['youtube']['channel_id']
- youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
- developerKey=DEVELOPER_KEY)
+ youtube = build(youtube_api, api_version, developerKey=key)
current_app.logger.info('[YOUTUBE] Updating %s', section)
search_response = youtube.search().list(
q="",
- channelId=youtube_channelID,
+ channelId=youtube_channel_id,
part='id,snippet',
order='date',
- maxResults=5
- ).execute()
+ maxResults=5).execute()
for search_result in search_response.get('items', []):
trimtitle = search_result['snippet']['title'][0:150]
if search_result['id']['kind'] == 'youtube#video':
data = {
- "name": section,
- "icon": feed_config[section]['icon'],
- "title": trimtitle,
- "description": search_result['snippet']['description'],
- "type": "youtube",
- "url": "https://www.youtube.com/watch?v=" +
+ "name":
+ section,
+ "icon":
+ feed_config[section]['icon'],
+ "title":
+ trimtitle,
+ "description":
+ search_result['snippet']['description'],
+ "type":
+ "youtube",
+ "url":
+ "https://www.youtube.com/watch?v=" +
search_result['id']['videoId'],
- "date": dateutil.parser.parse(
- search_result['snippet']['publishedAt'])}
+ "date":
+ dateutil.parser.parse(
+ search_result['snippet']['publishedAt'])
+ }
- try:
- current_app.mongo.db.items.replace_one(
- {'title': trimtitle}, data, True)
- except Exception as e:
- current_app.logger.error('DB replace error %s', e)
- return render_template(
- "message.html", icon="frown", msg=str(e))
+ database.replace_one({'title': trimtitle}, data)
- return render_template(
- "message.html",
- icon="smile",
- msg="Youtube API updated!")
-
-#
-# This GoG import is just so nasty and needs fixed, Will
-# be enabling it until it is.
-#
+ return render_template("message.html", msg="Youtube API updated!")
-@bp.route('/gog', methods=('GET', 'POST'))
+@bp.route('/gog', methods=["GET"])
def gog_update():
from datetime import datetime
@@ -184,10 +155,7 @@ def gog_update():
query = "mediaType=game&system=Linux&limit=50&page=" + str(count)
game_data = get_gog_info(query)
if game_data is None:
- return render_template(
- "message.html",
- icon="frown",
- msg="GoG query error")
+ return render_template("message.html", msg="GoG query error")
for search_result in game_data['products']:
@@ -219,33 +187,24 @@ def gog_update():
"publisher": search_result['publisher'],
"category": search_result['category'],
"url": "https://www.gog.com" + search_result['url'],
- "date": dateutil.parser.parse(release_date)}
+ "date": dateutil.parser.parse(release_date)
+ }
- try:
- current_app.mongo.db.items.replace_one(
- {'title': search_result['title']}, data, True)
- except Exception as e:
- return render_template(
- "message.html", icon="frown", msg=str(e))
+ database.replace_one({'title': search_result['title']}, data)
count = count + 1
return render_template(
- "message.html",
- icon="smile",
- msg="GoG games updated!")
+ "message.html", icon="smile", msg="GoG games updated!")
def get_gog_info(query):
- import json
- import requests
-
- GOG_API_URL = "https://embed.gog.com/games/ajax/filtered?"
+ gog_api_url = "https://embed.gog.com/games/ajax/filtered?"
- response = requests.get(GOG_API_URL + query)
+ response = requests.get(gog_api_url + query)
if response.status_code == 200:
- return (json.loads(response.content.decode('utf-8')))
+ return json.loads(response.content.decode('utf-8'))
else:
return None
(DIR) diff --git a/linuxgaming/util.py b/linuxgaming/util.py
@@ -0,0 +1,13 @@
+import yaml
+from flask import (current_app)
+
+
+def load_yaml():
+ """Return the YAML parsed config file."""
+ try:
+ with open('config/feed_config.yaml', 'r') as ymlfile:
+ cfg = yaml.load(ymlfile)
+ except yaml.YAMLError as exc:
+ current_app.logger.error('YAML read error %s', exc)
+
+ return cfg