This is a little Python script that allows you to scrape comments from a subbreddit on reddit.com.
NOTE: insert the forum name in line 35.
import scrapy
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.selector import Selector
from reddit.items import CommentItem
from scrapy.contrib.loader import ItemLoader
from scrapy.contrib.loader.processor import MapCompose, TakeFirst, Compose, Join, Identity
from reddit import utils
def get_comments_count(x):
return utils.regex_extractor(r"(.*)\s+comment.*", x, 1)
def get_upvoted(x):
return utils.regex_extractor(r"\((.*)\s+upvoted", x, 1)
class CommentItemLoader(ItemLoader):
default_item_class = CommentItem
default_input_processor = MapCompose(lambda x: x.strip())
default_output_processor = Compose(TakeFirst(), lambda x: x.strip())
default_selector_class = Selector
textpost_out = Compose(Join(" "), lambda x: x.strip())
comments_out = Compose(TakeFirst(), get_comments_count, lambda x: x.strip())
upvoted_out = Compose(TakeFirst(), get_upvoted, lambda x: x.strip())
comment_out = Compose(Join(" "), lambda x: x.strip())
class ThreadsSpider(CrawlSpider):
name = "threads"
allowed_domains = ["reddit.com"]
start_urls = (
"https://www.reddit.com/r/[INSERT FORUM HERE]",
)
rules = (
Rule(LinkExtractor(restrict_xpaths=(".//div[@class='nav-buttons']")), follow=True),
Rule(LinkExtractor(restrict_xpaths=(".//div[@class='content']//p[@class='parent']/a[@class='title']")), callback="parse_item"),
)
def parse_item(self, response):
hxs = Selector(response)
thread = hxs.xpath(".//p[@class='title']/a/text()").extract()
op = hxs.xpath(".//div[contains(@class, 'self')]//p[@class='tagline']/a[contains(@class, 'author')]/text()").extract()
thread_date = hxs.xpath(".//div[contains(@class, 'self')]//p[@class='tagline']/time/@title").extract()
textpost = hxs.xpath(".//div[contains(@class, 'self')]//div[@class='md']//text()").extract()
comments = hxs.xpath(".//div[contains(@class, 'self')]//a[contains(@class, 'comments')]/text()").extract()
vote_points = hxs.xpath(".//div[@class='linkinfo']/div[@class='score']/span[@class='number']/text()").extract()
upvoted = hxs.xpath(".//div[@class='linkinfo']/div[@class='score']/text()").extract()
rows = hxs.xpath(".//div[@class='commentarea']//div[contains(@class, 'comment')]/div[contains(@class, 'entry')]")
for row in rows:
l = CommentItemLoader(item = CommentItem(), response = response)
l.add_value("url", response.url)
l.add_value("thread", thread)
l.add_value("op", op)
l.add_value("thread_date", thread_date)
l.add_value("textpost", textpost)
l.add_value("comments", comments)
l.add_value("vote_points", vote_points)
l.add_value("upvoted", upvoted)
l.add_value("comment", row.xpath(".//div[contains(@class, 'usertext-body')]//text()").extract())
l.add_value("user", row.xpath(".//p[@class='tagline']/a[contains(@class, 'author')]/text()").extract())
l.add_value("time", row.xpath(".//p[@class='tagline']/time/@title").extract())
yield l.load_item()