VladPolskiy/postgres-web
1
0
Fork 0
mirror of https://github.com/postgres/pgweb.git synced 2025-08-13 13:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make the RSS feeder use the django model framework, to be consistent.

Browse Source
This commit is contained in:
Magnus Hagander
2010-06-11 19:31:29 +02:00
parent ba9e37d9d6
commit 7934c64c85
1 changed files with 26 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tools/rss/fetch_rss_feeds.py
View File

@ -3,18 +3,27 @@
import feedparser import feedparser
import psycopg2 import psycopg2
import socket import socket
import sys
import os
from datetime import datetime
# Set up for accessing django
from django.core.management import setup_environ
sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../../pgweb'))
import settings
setup_environ(settings)
from core.models import ImportedRSSFeed, ImportedRSSItem
from django.db import transaction
# Set timeout for loading RSS feeds # Set timeout for loading RSS feeds
socket.setdefaulttimeout(20) socket.setdefaulttimeout(20)
transaction.enter_transaction_management()
db = psycopg2.connect('host=/tmp dbname=pgweb') transaction.managed()
curs = db.cursor() for importfeed in ImportedRSSFeed.objects.all():
curs.execute("SELECT id,internalname,url FROM core_importedrssfeed")
for id,internalname,url in curs.fetchall():
try: try:
feed = feedparser.parse(url) feed = feedparser.parse(importfeed.url)
if not hasattr(feed, 'status'): if not hasattr(feed, 'status'):
# bozo_excpetion can seemingly be set when there is no error as well, # bozo_excpetion can seemingly be set when there is no error as well,
@ -25,16 +34,15 @@ for id,internalname,url in curs.fetchall():
if feed.status != 200: if feed.status != 200:
raise Exception('Feed returned status %s' % feed.status) raise Exception('Feed returned status %s' % feed.status)
for entry in feed.entries: for entry in feed.entries:
curs.execute("""INSERT INTO core_importedrssitem (feed_id, title, url, posttime) try:
SELECT %(feed)s, %(title)s, %(url)s, %(posttime)s item = ImportedRSSItem.objects.get(feed=importfeed, url=entry.link)
WHERE NOT EXISTS (SELECT * FROM core_importedrssitem c2 WHERE c2.feed_id=%(feed)s AND c2.url=%(url)s)""", { except ImportedRSSItem.DoesNotExist:
'feed': id, item = ImportedRSSItem(feed=importfeed,
'title': entry.title, title=entry.title[:100],
'url': entry.link, url=entry.link,
'posttime': entry.date, posttime=datetime.strptime(entry.date, "%a, %d %b %Y %H:%M:%S %Z"),
}) )
item.save()
transaction.commit()
except Exception, e: except Exception, e:
print "Failed to load %s: %s" % (internalname, e) print "Failed to load %s: %s" % (importfeed, e)
db.commit()