changed:
-
This is `roundup-feeder', an RSS issue feeder for Roundup. It creates a
new issue for each new item in its RSS feed, and keeps track of items
already entered using a pickled list.
I owe quite a bit of roundup-feeder to the chaps who built `spycyroll'
(http://spycyroll.sourceforge.net/). It's a neat program. Rather than
have multiple webpages to look at, though, I thought it would be nice if
everything were entered into my Roundup tracker.
You'll need to install Mark Pilgrim's `feedparser'
(http://feedparser.sourceforge.net), and of course you'll need to have
roundup installed too.
Setup is fairly simple. Set the roundup_* variables in config.ini (see
below), list the RSS feeds you want to watch, and go. I threw together
roundup-feeder to track security advisories & news, hence the selection
of feeds you see in the sample config.ini.
Modifying the program to handle your custom Roundup database should be
trivial - have a look at the enterNewsItem method of the RoundupFeeder
class.
You should probably set this up to run every 30 minutes or so via cron.
- Cian
config.ini ::
##
# File: config.ini
# Desc: A roundup-feeder sample config. I threw together roundup-feeder to
# track security advisories & news, hence the selection of feeds you see
# here.
# Auth: Cian Synnott <cian@dmz.ie>
# $Id: config.ini,v 1.2 2004/09/06 16:28:41 cian Exp $
##
[DEFAULT]
# The file in which to save our pickled list of items already entered
statefile = itemlist.sav
# Set the location of your roundup tracker
roundup-home = /path/to/roundup/tracker
# Set the user you want to enter issues into the tracker as
roundup-user = feeder
# Set the class of roundup issues - most likely `issue' :o)
roundup-class = issue
# Each feed has this layout - the feed link is the section header, and we can
# specify a name in the section.
[http://www.securityfocus.com/rss/vulnerabilities.xml]
name = SecurityFocus
[http://www.osvdb.org/backend/rss.php]
name = OSVDB
[http://www.packetstormsecurity.nl/whatsnew50.xml]
name = PacketStorm
[http://www.k-otik.com/advisories.xml]
name = k-otik advisories
[http://www.k-otik.com/exploits.xml]
name = k-otik exploits
[http://www.microsoft.com/technet/security/bulletin/secrss.aspx]
name = MS Bulletins
[http://www.djeaux.com/rss/insecure-vulnwatch.rss]
name = Vulnwatch
[http://msdn.microsoft.com/security/rss.xml]
name = MS Developer
[http://www.nwfusion.com/rss/security.xml]
name = nwfusion
[http://www.us-cert.gov/channels/techalerts.rdf]
name = US-CERT tech alerts
[http://www.us-cert.gov/channels/bulletins.rdf]
name = US-CERT bulletins
roundup-feeder ::
#!/usr/bin/env python
##
# File: roundup-feeder
# Desc: RSS issue feeder for roundup. It is partially derived from the
# `spycyroll' RSS aggregator, from http://spycyroll.sourceforge.net/.
#
# I've made it as straightforward as possible.
#
# Creates new roundup issues for each new item in its RSS feed, and keeps
# track of those already entered in the roundup DB in a pickled list.
#
# Auth: Cian Synnott <cian@dmz.ie>
# $Id: roundup-feeder.py,v 1.2 2004/09/06 21:30:12 cian Exp $
##
import sys
import pickle
# Mark Pilgrim's rather relaxed RSS parser (http://feedparser.sourceforge.net/)
import feedparser
# Ease of configuration
from ConfigParser import ConfigParser
# The all-important roundup stuff
import roundup.instance
from roundup import date
DEFAULT_CONFIG_FILE = "config.ini"
class Channel:
"""An RSS channel.
Allows an RSS channel to be loaded and stored in memory
rss : url to the RSS or RDF file. From this url, it figures out the rest;
title : Title as specified in RSS file UNLESS you specify it while creating
the object
link : url for the site, as specified in the RSS file
description : optional textual description.
items[]: collection of type NewsItem for the items in the feed
"""
def __init__(self, rss, title=None):
self.rss = rss
self.title = title
self.link = None
self.description = None
self.items = []
def load(self, rss=None):
"""Downloads and parses a channel
sets the feed's title, link and description.
sets and returns items[], for NewsItems defined
"""
if rss is None:
rss = self.rss
prss = feedparser.parse(rss)
channel = prss['channel']
items = prss['items']
if 'link' in channel.keys():
self.link = channel['link']
else:
self.link = ''
title = channel['title']
self.description = channel['description']
if self.title is None:
self.title = title
self.items = []
for item in items:
self.items.append(NewsItem(item))
return self.items
class NewsItem:
"""Each item in a channel"""
def __init__(self, dict):
self.link = dict.get('link', '')
self.title = dict['title']
self.description = dict['description']
if 'date' in dict.keys():
self.date = dict['date']
else:
self.date = None
def loadChannels(config):
"""Loads all channels in a configuration
"""
feeds = {}
for f in config.sections():
if config.has_option(f, 'name'):
feeds[f] = config.get(f, 'name')
else:
feeds[f] = None
channels = []
for f in feeds.keys():
c = Channel(f, feeds[f])
try:
c.load()
except:
continue
channels.append(c)
return channels
class RoundupFeeder:
"""A roundup RSS feeder class
A class for maintaining a `connection' to the roundup database and feeding
issues into it.
instance : An instance of the roundup tracker we're dealing with
db : The roundup database belonging to that instance
cl : The roundup class we enter issues as
uid : The user id we've opend the db as
"""
def __init__(self, home, user, klass):
"""Initialise the RoundupFeeder
home : The location of the roundup tracker on the filesystem
user : The user to open the database as
klass : The name of the `issue' class in the database
"""
# Connect to our roundup database
self.instance = roundup.instance.open(home)
self.db = self.instance.open('admin')
# First lookup and reconnect as this user
try:
self.uid = self.db.user.lookup(user)
username = self.db.user.get(self.uid, 'username')
self.db.close()
self.db = self.instance.open(username)
except:
print '''
Cannot open the tracker "%s" with username "%s".
Are you sure this user exists in the database?
'''%(home, user)
sys.exit(1)
try:
self.cl = self.db.getclass(klass)
except:
print '''
It appears that the configured Roundup class "%s" does not exist in the
database. Valid class names are: %s
'''%(klass, ', '.join(self.db.getclasses()))
sys.exit(1)
def cleanup(self):
"""Cleans up the RoundupFeeder
Closes the database we've been dealing with. This will need to be called
once you have created a RoundupFeeder; perhaps use a try: finally:
structure.
"""
self.db.close()
def enterNewsItem(self, channel, item):
"""Enter a news item as a roundup issue
Each news item should be a new issue in roundup.
channel : The channel this item is from
item : The item to enter as an issue
"""
# Setup issue title and issue content
title = item.title
if item.description:
content = '''
From: %s (%s)
%s
Link: %s
'''%(channel.title, channel.link, item.description, item.link)
else:
content = '''
From: %s (%s)
Link: %s
'''%(channel.title, channel.link, item.link)
# Set up the issue
issue = {}
issue['title'] = title
issue['files'] = []
issue['nosy'] = []
issue['superseder'] = []
# Set up the message
msg = {}
msg['author'] = self.uid
msg['date'] = date.Date('.')
msg['summary'] = title
msg['content'] = content
msg['files'] = []
msg['recipients'] = []
msg['messageid'] = ''
msg['inreplyto'] = ''
# My issues have a 'type' property for the issue type - I set that to
# roundup
if self.cl.getprops().has_key('type'):
issue['type'] = 'rssfeed'
# My issues have a 'status' property - I set that to unread
if self.cl.getprops().has_key('type'):
issue['status'] = 'unread'
# Now create new message & issue for this item
try:
message_id = self.db.msg.create(**msg)
issue['messages'] = [message_id]
nodeid = self.cl.create(**issue)
except Exception, inst:
print '''
Error creating issue for item '%s'.
'''%(item.link)
print inst
self.db.commit()
if __name__ == '__main__':
if len(sys.argv) > 1:
config_file = sys.argv[1]
else:
config_file = DEFAULT_CONFIG_FILE
config = ConfigParser()
config.readfp(open(config_file))
statefile = config.get('DEFAULT', 'statefile' )
roundup_home = config.get('DEFAULT', 'roundup-home' )
roundup_user = config.get('DEFAULT', 'roundup-user' )
roundup_class = config.get('DEFAULT', 'roundup-class')
channels = loadChannels(config)
try:
fp = open(statefile, 'r')
olditems = pickle.load(fp)
fp.close()
except:
olditems = []
newitems = []
feeder = RoundupFeeder(roundup_home, roundup_user, roundup_class)
# Now use try: finally: to make sure the database gets closed
try:
for c in channels:
for i in c.items:
if i.link not in newitems:
newitems.append(i.link)
if i.link not in olditems:
feeder.enterNewsItem(c, i)
finally:
feeder.cleanup()
# Now pickle our list of RSS items for the next run.
try:
fp = open(statefile, 'w')
pickle.dump(newitems, fp)
fp.close()
except:
print "Couldn't dump statefile!"