more refactoring - linuxgaming - Linux gaming aggregate tool, built to test out NodeJS.
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
---
(DIR) commit b11ff41ea03339b4f740e2bc49fe3a718623321d
(DIR) parent 5917aef9ef70f99064dd0db3711c4dff95a73db4
(HTM) Author: Jay Scott <me@jay.scot>
Date: Wed, 18 Jul 2018 20:18:33 +0100
more refactoring
Diffstat:
M config/feed_config.yaml | 11 +++++------
M linuxgaming/__init__.py | 6 +++---
M linuxgaming/database.py | 6 +++---
M linuxgaming/update.py | 103 ++++++++++++++-----------------
M linuxgaming/util.py | 36 ++++++++++++++++++++++++++++---
5 files changed, 91 insertions(+), 71 deletions(-)
---
(DIR) diff --git a/config/feed_config.yaml b/config/feed_config.yaml
@@ -18,14 +18,14 @@ NuSuey:
twitch:
website: "https://www.twitch.tv/nusuey/"
twitch_id: "7863182"
-
+
HexDSL:
icon: "hexdsl.png"
about: "Linux Gaming Rambles, shows and streams presented by HexDSL. Apparently you don't get a lot of characters in this box. so I'll just say that I am HexDSL (Sexy Hexy) and I make videos about gaming using Linux. I also Host the .XPenguin PodCast/VideoCast that people seem to like :) You can find more of me on twitch.tv/hexdsl/ thanks for taking the time to read this."
youtube:
website: "https://www.youtube.com/user/hexdsl/"
channel_id: "UCRE3NFNtdjR96-H4QG4U1Fg"
- twitch:
+ twitch:
website: "https://www.twitch.tv/hexdsl/"
twitch_id: "40594186"
@@ -35,7 +35,7 @@ TheLinuxGamer:
youtube:
website: "https://www.youtube.com/user/tuxreviews/"
channel_id: "UCv1Kcz-CuGM6mxzL3B1_Eiw"
-
+
GamingOnLinux:
icon: "gol.png"
about: "A little bit of information on who GamingOnLinux are, we are just a small group of passionate Linux fans who wish to bring you nothing but the best in Linux gaming news. Started by liamdawe on his own in 2009 on a .info address, we later switched to using a .com domain name."
@@ -46,7 +46,7 @@ GamingOnLinux:
twitch:
website: "https://www.twitch.tv/gamingonlinux/"
twitch_id: "50905707"
-
+
BLGP:
icon: "blgp.png"
about: "The Best Linux Games Podcast features only the best news, reviews, and deals exclusively focusing on titles available for the Linux OS. Heavily focuses on Steam titles. Pushed onto teh interwebs every Saturday and hosted by Skookiesprite, the podcast brings an entire week's worth of news, reviews, tech tips, and the best-loved DEALS! Segment straight into your brain."
@@ -80,4 +80,4 @@ XPenguin:
rss:
website: "http://xpenguin.club/"
type: "podcast"
- url: "http://xpenguin.club/rss.xml"
-\ No newline at end of file
+ url: "http://xpenguin.club/rss.xml"
(DIR) diff --git a/linuxgaming/__init__.py b/linuxgaming/__init__.py
@@ -46,13 +46,13 @@ def create_app():
def internal_error(error):
app.logger.error('internal error %s', error)
return render_template(
- "message.html", icon="frown", msg="Something went wrong!"), 500
+ "message.html", msg="Something went wrong!"), 500
@app.errorhandler(404)
- def page_not_found():
+ def page_not_found(page):
app.logger.info('page not found')
return render_template(
- "message.html", icon="frown", msg="I think you are lost!"), 404
+ "message.html", msg="I think you are lost!"), 404
@app.template_filter('strftime')
def _jinja2_filter_datetime(date):
(DIR) diff --git a/linuxgaming/database.py b/linuxgaming/database.py
@@ -1,4 +1,4 @@
-from flask import current_app
+from flask import current_app, abort
def find_all(query={}):
@@ -7,7 +7,7 @@ def find_all(query={}):
d = current_app.mongo.db.items.find(query).sort('date', -1)
except Exception as e:
current_app.logger.error('DB replace error %s', e)
- return False
+ abort(500)
return d
@@ -18,6 +18,6 @@ def replace_one(query, data, upsert=True):
current_app.mongo.db.items.replace_one(query, data, upsert)
except Exception as e:
current_app.logger.error('DB replace error %s', e)
- return False
+ abort(500)
return True
(DIR) diff --git a/linuxgaming/update.py b/linuxgaming/update.py
@@ -1,52 +1,60 @@
from googleapiclient.discovery import build
from twitch import TwitchClient
from flask import Blueprint, render_template, current_app
-from . import database
-import json
-import requests
-import feedparser
import dateutil.parser
+from . import database
from . import util
bp = Blueprint('update', __name__, url_prefix='/update')
-def parse(url):
- return feedparser.parse(url).entries
-
-
@bp.route('/rss', methods=["GET"])
def rss_update():
+ # load sources config
feed_config = util.load_yaml()
for section in feed_config:
+ # if it does not have an rss section skip
if 'rss' not in feed_config[section]:
continue
current_app.logger.info('[RSS] Updating %s', section)
- feeds = parse(feed_config[section]['rss']['url'])
+
+ # parse the source url rss feed
+ feeds = util.feed_parse(feed_config[section]['rss']['url'])
+
+ # check for errors
+ if feeds is None:
+ continue
for feed in feeds:
- trimtitle = feed.title[0:150]
+ # if not title, then just skip
+ if hasattr(feed, 'title'):
+ trimmed_title = feed.title[0:150]
+ else:
+ continue
+ # some feeds dont have a description, RSS 2.0
if not hasattr(feed, 'description'):
description = ""
else:
description = feed.description
+ # construct db item
data = {
"name": section,
"icon": feed_config[section]['icon'],
- "title": trimtitle,
+ "title": trimmed_title,
"description": description,
"url": feed.link,
"type": feed_config[section]['rss']['type'],
"date": dateutil.parser.parse(feed.updated)
}
- database.replace_one({'title': trimtitle}, data)
+ # insert based on title
+ database.replace_one({'title': trimmed_title}, data)
return render_template("message.html", msg="RSS feeds updated!")
@@ -61,6 +69,7 @@ def twitch_update():
continue
current_app.logger.info('[TWITCH] Updating %s', section)
+
twitch_channelid = feed_config[section]['twitch']['twitch_id']
client = TwitchClient(
@@ -77,11 +86,11 @@ def twitch_update():
)
for search_results in videos:
- trimtitle = search_results['title'][0:150]
+ trimmed_title = search_results['title'][0:150]
data = {
"name": section,
"icon": feed_config[section]['icon'],
- "title": trimtitle,
+ "title": trimmed_title,
"description": search_results['description'],
"url": search_results['url'],
"type": "twitch",
@@ -96,10 +105,6 @@ def twitch_update():
@bp.route('/youtube', methods=["GET"])
def youtube_update():
- key = current_app.config['YOUTUBE_APIKEY']
- youtube_api = 'youtube'
- api_version = 'v3'
-
feed_config = util.load_yaml()
for section in feed_config:
@@ -108,9 +113,11 @@ def youtube_update():
youtube_channel_id = feed_config[section]['youtube']['channel_id']
- youtube = build(youtube_api, api_version, developerKey=key)
+ youtube = build(
+ 'youtube', 'v3', developerKey=current_app.config['YOUTUBE_APIKEY'])
current_app.logger.info('[YOUTUBE] Updating %s', section)
+
search_response = youtube.search().list(
q="",
channelId=youtube_channel_id,
@@ -119,28 +126,26 @@ def youtube_update():
maxResults=5).execute()
for search_result in search_response.get('items', []):
- trimtitle = search_result['snippet']['title'][0:150]
- if search_result['id']['kind'] == 'youtube#video':
- data = {
- "name":
- section,
- "icon":
- feed_config[section]['icon'],
- "title":
- trimtitle,
- "description":
- search_result['snippet']['description'],
- "type":
- "youtube",
- "url":
- "https://www.youtube.com/watch?v=" +
- search_result['id']['videoId'],
- "date":
- dateutil.parser.parse(
- search_result['snippet']['publishedAt'])
- }
-
- database.replace_one({'title': trimtitle}, data)
+ trimmed_title = search_result['snippet']['title'][0:150]
+ data = {
+ "name":
+ section,
+ "icon":
+ feed_config[section]['icon'],
+ "title":
+ trimmed_title,
+ "description":
+ search_result['snippet']['description'],
+ "type":
+ "youtube",
+ "url":
+ "https://www.youtube.com/watch?v=" +
+ search_result['id']['videoId'],
+ "date":
+ dateutil.parser.parse(search_result['snippet']['publishedAt'])
+ }
+
+ database.replace_one({'title': trimmed_title}, data)
return render_template("message.html", msg="Youtube API updated!")
@@ -153,9 +158,7 @@ def gog_update():
count = 1
while count < 51:
query = "mediaType=game&system=Linux&limit=50&page=" + str(count)
- game_data = get_gog_info(query)
- if game_data is None:
- return render_template("message.html", msg="GoG query error")
+ game_data = util.get_gog_info(query)
for search_result in game_data['products']:
@@ -196,15 +199,3 @@ def gog_update():
return render_template(
"message.html", icon="smile", msg="GoG games updated!")
-
-
-def get_gog_info(query):
-
- gog_api_url = "https://embed.gog.com/games/ajax/filtered?"
-
- response = requests.get(gog_api_url + query)
-
- if response.status_code == 200:
- return json.loads(response.content.decode('utf-8'))
- else:
- return None
(DIR) diff --git a/linuxgaming/util.py b/linuxgaming/util.py
@@ -1,13 +1,43 @@
import yaml
-from flask import (current_app)
+import feedparser
+import json
+import requests
+from flask import (current_app, abort)
def load_yaml():
"""Return the YAML parsed config file."""
+
try:
with open('config/feed_config.yaml', 'r') as ymlfile:
cfg = yaml.load(ymlfile)
- except yaml.YAMLError as exc:
- current_app.logger.error('YAML read error %s', exc)
+ except (yaml.YAMLError, FileNotFoundError) as e:
+ current_app.logger.error('YAML read error %s', e)
+ abort(500)
return cfg
+
+
+def feed_parse(url):
+
+ # parse the feed and get the results
+ res = feedparser.parse(url)
+
+ if res.entries:
+ return res.entries
+ else:
+ current_app.logger.error('FEED parse error %s', url)
+
+ return None
+
+
+def get_gog_info(query):
+
+ gog_api_url = "https://embed.gog.com/games/ajax/filtered?"
+
+ response = requests.get(gog_api_url + query)
+
+ if response.status_code == 200:
+ return json.loads(response.content.decode('utf-8'))
+ else:
+ abort(500)