* Try to handle relative links in aggregated feeds. However,
[ikiwiki] / IkiWiki / Plugin / aggregate.pm
1 #!/usr/bin/perl
2 # Blog aggregation plugin.
3 package IkiWiki::Plugin::aggregate;
4
5 use warnings;
6 use strict;
7 use IkiWiki;
8 use HTML::Entities;
9 use HTML::Parser;
10 use HTML::Tagset;
11 use URI;
12
13 my %feeds;
14 my %guids;
15
16 sub import { #{{{
17         IkiWiki::hook(type => "getopt", id => "aggregate", 
18                 call => \&getopt);
19         IkiWiki::hook(type => "checkconfig", id => "aggregate",
20                 call => \&checkconfig);
21         IkiWiki::hook(type => "filter", id => "aggregate", 
22                 call => \&filter);
23         IkiWiki::hook(type => "preprocess", id => "aggregate",
24                 call => \&preprocess);
25         IkiWiki::hook(type => "delete", id => "aggregate",
26                 call => \&delete);
27         IkiWiki::hook(type => "savestate", id => "aggregate",
28                 call => \&savestate);
29 } # }}}
30
31 sub getopt () { #{{{
32         eval q{use Getopt::Long};
33         Getopt::Long::Configure('pass_through');
34         GetOptions("aggregate" => \$IkiWiki::config{aggregate});
35 } #}}}
36
37 sub checkconfig () { #{{{
38         loadstate();
39         if ($IkiWiki::config{aggregate}) {
40                 IkiWiki::loadindex();
41                 aggregate();
42                 savestate();
43         }
44 } #}}}
45
46 sub filter (@) { #{{{
47         my %params=@_;
48         my $page=$params{page};
49
50         # Mark all feeds originating on this page as removable;
51         # preprocess will unmark those that still exist.
52         remove_feeds($page);
53
54         return $params{content};
55 } # }}}
56
57 sub preprocess (@) { #{{{
58         my %params=@_;
59
60         foreach my $required (qw{name url}) {
61                 if (! exists $params{$required}) {
62                         return "[[aggregate plugin missing $required parameter]]";
63                 }
64         }
65
66         my $feed={};
67         my $name=$params{name};
68         if (exists $feeds{$name}) {
69                 $feed=$feeds{$name};
70         }
71         else {
72                 $feeds{$name}=$feed;
73         }
74         $feed->{name}=$name;
75         $feed->{sourcepage}=$params{page};
76         $feed->{url}=$params{url};
77         my $dir=exists $params{dir} ? $params{dir} : $params{page}."/".IkiWiki::titlepage($params{name});
78         $dir=~s/^\/+//;
79         ($dir)=$dir=~/$IkiWiki::config{wiki_file_regexp}/;
80         $feed->{dir}=$dir;
81         $feed->{feedurl}=defined $params{feedurl} ? $params{feedurl} : "";
82         $feed->{updateinterval}=defined $params{updateinterval} ? $params{updateinterval} * 60 : 15 * 60;
83         $feed->{expireage}=defined $params{expireage} ? $params{expireage} : 0;
84         $feed->{expirecount}=defined $params{expirecount} ? $params{expirecount} : 0;
85         delete $feed->{remove};
86         $feed->{lastupdate}=0 unless defined $feed->{lastupdate};
87         $feed->{numposts}=0 unless defined $feed->{numposts};
88         $feed->{newposts}=0 unless defined $feed->{newposts};
89         $feed->{message}="new feed" unless defined $feed->{message};
90         $feed->{tags}=[];
91         while (@_) {
92                 my $key=shift;
93                 my $value=shift;
94                 if ($key eq 'tag') {
95                         push @{$feed->{tags}}, $value;
96                 }
97         }
98
99         return "<a href=\"".$feed->{url}."\">".$feed->{name}."</a>: ".
100                "<i>".$feed->{message}."</i> (".$feed->{numposts}.
101                " stored posts; ".$feed->{newposts}." new)<br />";
102 } # }}}
103
104 sub delete (@) { #{{{
105         my @files=@_;
106
107         # Remove feed data for removed pages.
108         foreach my $file (@files) {
109                 my $page=IkiWiki::pagename($file);
110                 remove_feeds($page);
111         }
112 } #}}}
113
114 sub loadstate () { #{{{
115         if (-e "$IkiWiki::config{wikistatedir}/aggregate") {
116                 open (IN, "$IkiWiki::config{wikistatedir}/aggregate" ||
117                         die "$IkiWiki::config{wikistatedir}/aggregate: $!");
118                 while (<IN>) {
119                         $_=IkiWiki::possibly_foolish_untaint($_);
120                         chomp;
121                         my $data={};
122                         foreach my $i (split(/ /, $_)) {
123                                 my ($field, $val)=split(/=/, $i, 2);
124                                 if ($field eq "name" || $field eq "feed" ||
125                                     $field eq "guid" || $field eq "message") {
126                                         $data->{$field}=decode_entities($val, " \t\n");
127                                 }
128                                 elsif ($field eq "tag") {
129                                         push @{$data->{tags}}, $val;
130                                 }
131                                 else {
132                                         $data->{$field}=$val;
133                                 }
134                         }
135                         
136                         if (exists $data->{name}) {
137                                 $feeds{$data->{name}}=$data;
138                         }
139                         elsif (exists $data->{guid}) {
140                                 $guids{$data->{guid}}=$data;
141                         }
142                 }
143
144                 close IN;
145         }
146 } #}}}
147
148 sub savestate () { #{{{
149         eval q{use HTML::Entities};
150         die $@ if $@;
151         open (OUT, ">$IkiWiki::config{wikistatedir}/aggregate" ||
152                 die "$IkiWiki::config{wikistatedir}/aggregate: $!");
153         foreach my $data (values %feeds, values %guids) {
154                 if ($data->{remove}) {
155                         if ($data->{name}) {
156                                 foreach my $guid (values %guids) {
157                                         if ($guid->{feed} eq $data->{name}) {
158                                                 $guid->{remove}=1;
159                                         }
160                                 }
161                         }
162                         else {
163                                 unlink pagefile($data->{page});
164                         }
165                         next;
166                 }
167
168                 my @line;
169                 foreach my $field (keys %$data) {
170                         if ($field eq "name" || $field eq "feed" ||
171                             $field eq "guid" || $field eq "message") {
172                                 push @line, "$field=".encode_entities($data->{$field}, " \t\n");
173                         }
174                         elsif ($field eq "tags") {
175                                 push @line, "tag=$_" foreach @{$data->{tags}};
176                         }
177                         else {
178                                 push @line, "$field=".$data->{$field};
179                         }
180                 }
181                 print OUT join(" ", @line)."\n";
182         }
183         close OUT;
184 } #}}}
185
186 sub aggregate () { #{{{
187         eval q{use XML::Feed};
188         die $@ if $@;
189         eval q{use HTML::Entities};
190         die $@ if $@;
191
192         foreach my $feed (values %feeds) {
193                 next unless $IkiWiki::config{rebuild} || 
194                         time - $feed->{lastupdate} >= $feed->{updateinterval};
195                 $feed->{lastupdate}=time;
196                 $feed->{newposts}=0;
197                 $IkiWiki::forcerebuild{$feed->{sourcepage}}=1;
198
199                 IkiWiki::debug("checking feed ".$feed->{name}." ...");
200
201                 if (! length $feed->{feedurl}) {
202                         my @urls=XML::Feed->find_feeds($feed->{url});
203                         if (! @urls) {
204                                 $feed->{message}="could not find feed at ".$feed->{feedurl};
205                                 IkiWiki::debug($feed->{message});
206                                 next;
207                         }
208                         $feed->{feedurl}=pop @urls;
209                 }
210                 my $f=eval{XML::Feed->parse(URI->new($feed->{feedurl}))};
211                 if ($@) {
212                         $feed->{message}="feed crashed XML::Feed! $@";
213                         IkiWiki::debug($feed->{message});
214                         next;
215                 }
216                 if (! $f) {
217                         $feed->{message}=XML::Feed->errstr;
218                         IkiWiki::debug($feed->{message});
219                         next;
220                 }
221
222                 foreach my $entry ($f->entries) {
223                         add_page(
224                                 feed => $feed,
225                                 title => defined $entry->title ? decode_entities($entry->title) : "untitled",
226                                 link => $entry->link,
227                                 content => $entry->content->body,
228                                 guid => defined $entry->id ? $entry->id : time."_".$feed->name,
229                                 ctime => $entry->issued ? ($entry->issued->epoch || time) : time,
230                         );
231                 }
232
233                 $feed->{message}="processed ok";
234         }
235
236         # TODO: expiry
237 } #}}}
238
239 sub add_page (@) { #{{{
240         my %params=@_;
241         
242         my $feed=$params{feed};
243         my $guid={};
244         my $mtime;
245         if (exists $guids{$params{guid}}) {
246                 # updating an existing post
247                 $guid=$guids{$params{guid}};
248         }
249         else {
250                 # new post
251                 $guid->{guid}=$params{guid};
252                 $guids{$params{guid}}=$guid;
253                 $mtime=$params{ctime};
254                 $feed->{numposts}++;
255                 $feed->{newposts}++;
256
257                 # assign it an unused page
258                 my $page=IkiWiki::titlepage($params{title});
259                 # escape slashes and periods in title so it doesn't specify
260                 # directory name or trigger ".." disallowing code.
261                 $page=~s!([/.])!"__".ord($1)."__"!eg;
262                 $page=$feed->{dir}."/".$page;
263                 $page=lc($page);
264                 ($page)=$page=~/$IkiWiki::config{wiki_file_regexp}/;
265                 if (! defined $page || ! length $page) {
266                         $page=$feed->{dir}."/item";
267                 }
268                 my $c="";
269                 while (exists $IkiWiki::pagesources{$page.$c} ||
270                        -e pagefile($page.$c)) {
271                         $c++
272                 }
273                 $guid->{page}=$page;
274                 IkiWiki::debug("creating new page $page");
275         }
276         $guid->{feed}=$feed->{name};
277         
278         # To write or not to write? Need to avoid writing unchanged pages
279         # to avoid unneccessary rebuilding. The mtime from rss cannot be
280         # trusted; let's use a digest.
281         eval q{use Digest::MD5 'md5_hex'};
282         require Encode;
283         my $digest=md5_hex(Encode::encode_utf8($params{content}));
284         return unless ! exists $guid->{md5} || $guid->{md5} ne $digest || $IkiWiki::config{rebuild};
285         $guid->{md5}=$digest;
286
287         # Create the page.
288         my $template=IkiWiki::template("aggregatepost.tmpl", blind_cache => 1);
289         $template->param(title => $params{title})
290                 if defined $params{title} && length($params{title});
291         $template->param(content => htmlescape(htmlabs($params{content}, $feed->{feedurl})));
292         $template->param(url => $feed->{url});
293         $template->param(name => $feed->{name});
294         $template->param(link => urlabs($params{link}, $feed->{feedurl}))
295                 if defined $params{link};
296         if (ref $feed->{tags}) {
297                 $template->param(tags => [map { tag => $_ }, @{$feed->{tags}}]);
298         }
299         IkiWiki::writefile($guid->{page}.".html", $IkiWiki::config{srcdir},
300                 $template->output);
301
302         # Set the mtime, this lets the build process get the right creation
303         # time on record for the new page.
304         utime $mtime, $mtime, pagefile($guid->{page}) if defined $mtime;
305 } #}}}
306
307 sub htmlescape ($) { #{{{
308         # escape accidental wikilinks and preprocessor stuff
309         my $html=shift;
310         $html=~s/(?<!\\)\[\[/\\\[\[/g;
311         return $html;
312 } #}}}
313
314 sub urlabs ($$) { #{{{
315         my $url=shift;
316         my $urlbase=shift;
317
318         URI->new_abs($url, $urlbase)->as_string;
319 } #}}}
320
321 sub htmlabs ($$) { #{{{
322         # Convert links in html from relative to absolute.
323         # Note that this is a heuristic, which is not specified by the rss
324         # spec and may not be right for all feeds. Also, see Debian
325         # bug #XXXX TODO: get bug.
326         my $html=shift;
327         my $urlbase=shift;
328
329         my $ret="";
330         my $p = HTML::Parser->new(api_version => 3);
331         $p->handler(default => sub { $ret.=join("", @_) }, "text");
332         $p->handler(start => sub {
333                 my ($tagname, $pos, $text) = @_;
334                 if (ref $HTML::Tagset::linkElements{$tagname}) {
335                         while (4 <= @$pos) {
336                                 # use attribute sets from right to left
337                                 # to avoid invalidating the offsets
338                                 # when replacing the values
339                                 my($k_offset, $k_len, $v_offset, $v_len) =
340                                         splice(@$pos, -4);
341                                 my $attrname = lc(substr($text, $k_offset, $k_len));
342                                 next unless grep { $_ eq $attrname } @{$HTML::Tagset::linkElements{$tagname}};
343                                 next unless $v_offset; # 0 v_offset means no value
344                                 my $v = substr($text, $v_offset, $v_len);
345                                 $v =~ s/^([\'\"])(.*)\1$/$2/;
346                                 my $new_v=urlabs($v, $urlbase);
347                                 $new_v =~ s/\"/&quot;/g; # since we quote with ""
348                                 substr($text, $v_offset, $v_len) = qq("$new_v");
349                         }
350                 }
351                 $ret.=$text;
352         }, "tagname, tokenpos, text");
353         $p->parse($html);
354         $p->eof;
355
356         return $ret;
357 } #}}}
358
359 sub remove_feeds () { #{{{
360         my $page=shift;
361
362         my %removed;
363         foreach my $id (keys %feeds) {
364                 if ($feeds{$id}->{sourcepage} eq $page) {
365                         $feeds{$id}->{remove}=1;
366                         $removed{$id}=1;
367                 }
368         }
369 } #}}}
370
371 sub pagefile ($) { #{{{
372         my $page=shift;
373
374         return "$IkiWiki::config{srcdir}/$page.html";
375 } #}}}
376
377 1