User:Lupin/histdump.sh

From Wikipedia, the free encyclopedia

[edit] histdump.sh

Dump history of a page.

#!/bin/bash
# Usage: histdump.sh [-r] 'Title of wiki page'
# -r: download in chronological order instead of most recent first
# FIXME: deal with more than 500 entries
if [ "$1" = "-r" ]; then mung=tac; shift; else mung=cat; fi
page=$(echo "$1" | sed -e 's/ /_/g')
outputdir="$page"
mkdir "$outputdir"
sitebase='http://en.wikipedia.org'
histpage="${sitebase}/w/index.php?title=${page}&limit=500&action=history"
hist="$(wget -q -O- $histpage|grep '^<li>')"
numedits=$(echo "$hist"|wc -l)
i=0
echo "$hist" | "$mung" | while read line; do
    url=${sitebase}$(echo $line|sed -e 's/.*a href="\([^"]*oldid=[0-9]*\)" title="[^"]*">[0-9].*/\1/' -e 's/[&]amp;/\&/g')
    time=$(echo $line|sed -e 's/.*a href="\([^"]*oldid=[0-9]*\)" title="[^"]*">\([0-9][^<]*\).*/\2/'|sed -e 's/,//g')
    oldid=$(echo $url|sed -e 's/.*oldid=\([0-9]*\).*/\1/')
    user=$(echo $line|sed -e 's/.*class=.history-user[^>]*>[^<]*<a[^>]*>\([^<]*\).*/\1/')
    comment=$(echo $line|sed -e 's/.*class=.comment.>[(]\([^<]*\)[)].*/\1/')
    if [ "$comment" = "$line" ]; then comment=''; fi
    outputfile="$outputdir"/"$oldid";
    i=$((i+1));
    echo -e "Edit ${i} of ${numedits} by ${user}, oldid ${oldid}, time ${time}"
    (echo Page: "$page"; echo Editor: "$user"; echo Time: "$time"; echo Edit summary: "$comment") > "$outputfile"
    wget -q -O- "${sitebase}/w/index.php?title=${page}&action=raw&oldid=${oldid}" >> "$outputfile"
    touch --date="${time}" "$outputfile"
done 

[edit] diffgen.sh

Generate diffs from a set of dumped revisions. Usage: diffgen.sh rev1 rev2 ...

Uses the timestamps on the revision files to sort them.

#!/bin/bash
files=$(ls -Srt "$@")
DIFF="diff -u"
oldfile=
for newfile in $files ; do
    if [ ! -z "$oldfile" ]; then 
        $DIFF "$oldfile" "$newfile"
    fi
    oldfile=$newfile
done