Begin to re-orient to 'older' search based pagination

.gitignore

@@ -4,4 +4,5 @@ favicon.ico
 dist/
 www/assets/*.*
 data/DUMMY.json
+data/files/*
 pod2htmd.tmp

lib/Trog/Data/DUMMY.pm

@@ -25,10 +25,18 @@ our $datastore = 'data/DUMMY.json';
 sub lang { 'Perl Regex in Quotemeta' }
 sub help { 'https://perldoc.perl.org/functions/quotemeta.html' }
 
-sub read ($self, $count=0) {
+our $posts;
+
+sub read ($self, $query=undef) {
     confess "Can't find datastore!" unless -f $datastore;
     my $slurped = File::Slurper::read_text($datastore);
-    return JSON::MaybeXS::decode_json($slurped);
+    $posts = JSON::MaybeXS::decode_json($slurped);
+    return $posts;
+}
+
+sub count ($self) {
+    $posts //= $self->read();
+    return scalar(@$posts);
 }
 
 sub write($self,$data) {

lib/Trog/Data/FlatFile.pm

@@ -0,0 +1,65 @@
+package Trog::Data::FlatFile;
+
+use strict;
+use warnings;
+
+no warnings 'experimental';
+use feature qw{signatures};
+
+use Carp qw{confess};
+use JSON::MaybeXS;
+use File::Slurper;
+use File::Copy;
+use Mojo::File;
+use List::Util;
+
+use parent qw{Trog::DataModule};
+
+our $datastore = 'data/files/';
+sub lang { 'Perl Regex in Quotemeta' }
+sub help { 'https://perldoc.perl.org/functions/quotemeta.html' }
+our @index;
+
+sub read ($self, $query=undef) {
+    @index //= $self->_index();
+    my @items;
+    foreach my $item (@index) {
+        my $slurped = File::Slurper::read_text($item);
+        my $parsed  = JSON::MaybeXS::decode_json($slurped);
+        push(@items,$parsed) unless $self->filter($parsed);
+        last if scalar(@items) == $query->{limit};
+    }
+    return @items;
+}
+
+sub _index ($self) {
+    return @index if @index;
+    confess "Can't find datastore!" unless -d $datastore;
+    opendir(my $dh, $datastore) or die;
+    @index = grep { -f $_ } readdir $dh;
+    closedir $dh;
+    return @index;
+}
+
+sub write($self,$data) {
+    open(my $fh, '>', $datastore) or confess;
+    print $fh JSON::MaybeXS::encode_json($data);
+    close $fh;
+}
+
+sub count ($self) {
+    @index //= $self->_index();
+    return scalar(@index);
+}
+
+sub delete($self, @posts) {
+    my $example_posts = $self->read();
+    foreach my $update (@posts) {
+        @$example_posts = grep { $_->{id} ne $update->{id} } @$example_posts;
+    }
+    $self->write($example_posts);
+    return 0;
+}
+
+
+1;

lib/Trog/DataModule.pm

@@ -39,11 +39,11 @@ sub new ($class, $config) {
 }
 
 #It is required that subclasses implement this
-sub lang  ($self) { die 'stub' }
-sub help  ($self) { die 'stub' }
-#If count is passed, just return the total posts
-sub read  ($self,$count=0) { die 'stub' }
-sub write ($self) { die 'stub' }
+sub lang  ($self) { ... }
+sub help  ($self) { ... }
+sub read  ($self,$query=undef) { ... }
+sub write ($self) { ... }
+sub count ($self) { ... }
 
 =head1 METHODS
 
@@ -74,28 +74,37 @@ As implemented, this takes the data as a given and filters in post.
 
 sub get ($self, %request) {
 
-    my $example_posts = $self->read();
+    my $posts = $self->read();
+    my @filtered = $self->filter(\%request, @$posts);
+    @filtered = $self->_fixup(@filtered);
+    @filtered = $self->paginate(\%request,@filtered);
+    return @filtered;
+}
+
+sub _fixup ($self, @filtered) {
+    @filtered = _add_post_type(@filtered);
+    # Next, add the type of post this is
+    @filtered = _add_media_type(@filtered);
+    # Finally, add visibility
+    @filtered = _add_visibility(@filtered);
+    return @filtered;
+}
+
+sub filter ($self, $query, @filtered) {
+    my %request = %$query; #XXX update varnames instead
     $request{acls} //= [];
     $request{tags} //=[];
 
-    my @filtered = @$example_posts;
-
-    # If an ID is passed, just get that (and all it's prior versions
+    # If an ID is passed, just get that (and all it's prior versions)
     if ($request{id}) {
         @filtered = grep { $_->{id} eq $request{id} } @filtered if $request{id};
-
         @filtered = _dedup_versions($request{version}, @filtered);
-        @filtered = _add_post_type(@filtered);
-        # Next, add the type of post this is
-        @filtered = _add_media_type(@filtered);
-        # Finally, add visibility
-        @filtered = _add_visibility(@filtered);
         return (1, \@filtered);
     }
 
     @filtered = _dedup_versions(undef, @filtered);
 
-    # Heal bad data
+    #XXX Heal bad data -- probably not needed
     @filtered = map { my $t = $_->{tags}; @$t = grep { defined $_ } @$t; $_ } @filtered;
 
     # Next, handle the query, tags and ACLs
@@ -105,21 +114,15 @@ sub get ($self, %request) {
 
     @filtered = grep { $_->{user} eq $request{author} } @filtered if $request{author};
 
-    # Finally, paginate
+    return @filtered;
+}
+
+sub paginate ($self, $query, @filtered) {
+    my %request = %$query; #XXX change varnames
     my $offset = int($request{limit} // 25);
     $offset = @filtered < $offset ? @filtered : $offset;
-    my $pages = int(scalar(@filtered) / ($offset || 1) );
-
     @filtered = splice(@filtered, ( int($request{page}) -1) * $offset, $offset) if $request{page} && $request{limit};
-
-    # Next, go ahead and build the "post type"
-    @filtered = _add_post_type(@filtered);
-    # Next, add the type of post this is
-    @filtered = _add_media_type(@filtered);
-    # Finally, add visibility
-    @filtered = _add_visibility(@filtered);
-
-    return ($pages,\@filtered);
+    return @filtered;
 }
 
 sub _dedup_versions ($version=-1, @posts) {
@@ -179,18 +182,13 @@ sub _add_visibility (@posts) {
     } @posts;
 }
 
-=head2 total_posts() = INT $num
+=head2 count() = INT $num
 
 Returns the total number of posts.
 Used to determine paginator parameters.
 
 =cut
 
-sub total_posts ($self) {
-    my $example_posts = $self->read(1);
-    return scalar(@$example_posts);
-}
-
 =head2 add(@posts) = BOOL $failed_or_not
 
 Add the provided posts to the datastore.

lib/Trog/Routes/HTML.pm

@@ -499,11 +499,11 @@ sub post ($query, $render_cb) {
     return forbidden($query, $render_cb) unless grep { $_ eq 'admin' } @{$query->{acls}};
 
     my $tags  = _coerce_array($query->{tag});
-    my ($pages,$posts) = _post_helper($query, $tags, $query->{acls});
+    my @posts = _post_helper($query, $tags, $query->{acls});
     my $css   = _build_themed_styles('post.css');
     my $js    = _build_themed_scripts('post.js');
     push(@$css, '/styles/avatars.css');
-    my (undef, $acls) = _post_helper({}, ['series'], $query->{acls});
+    my @acls  = _post_helper({}, ['series'], $query->{acls});
 
     my $app = 'file';
     if ($query->{route}) {
@@ -514,10 +514,11 @@ sub post ($query, $render_cb) {
 
     #Filter displaying acl/visibility tags
     my @visibuddies = qw{public unlisted private};
-    foreach my $post (@$posts) {
-        @{$post->{tags}} = grep { my $tag = $_; !grep { $tag eq $_ } (@visibuddies, map { $_->{aclname} } @$acls ) } @{$post->{tags}};
+    foreach my $post (@posts) {
+        @{$post->{tags}} = grep { my $tag = $_; !grep { $tag eq $_ } (@visibuddies, map { $_->{aclname} } @acls ) } @{$post->{tags}};
     }
 
+    my $limit = int($query->{limit} || 25);
     return $render_cb->('post.tx', {
         title       => 'New Post',
         to          => $query->{to},
@@ -525,16 +526,17 @@ sub post ($query, $render_cb) {
         post_visibilities => \@visibuddies,
         stylesheets => $css,
         scripts     => $js,
-        posts       => $posts,
+        posts       => \@posts,
         can_edit    => 1,
         route       => $query->{route},
         category    => '/posts',
-        page        => int($query->{page} || 1),
-        limit       => int($query->{limit} || 25),
-        pages       => $pages,
+        limit       => $limit,
+        pages       => scalar(@posts) > $limit,
+        older       => $query->{older},
+        last        => $posts[-1]->{created},
         sizes       => [25,50,100],
         id          => $query->{id},
-        acls        => $acls,
+        acls        => \@acls,
         post        => { tags => $query->{tag} },
         edittype    => $query->{type} || 'microblog',
         app         => $app,
@@ -601,10 +603,10 @@ Add new 'series' (ACLs) to classify content with.
 
 sub series ($query, $render_cb) {
     #Grab the relevant tag (aclname), then pass that to posts
-    my (undef, $posts) = _post_helper($query, [], $query->{acls});
+    my @posts = _post_helper($query, [], $query->{acls});
     delete $query->{id};
 
-    $query->{tag} = $posts->[0]->{aclname};
+    $query->{tag} = $posts[0]->{aclname};
     return posts($query,$render_cb);
 }
 
@@ -622,10 +624,10 @@ sub avatars ($query, $render_cb) {
     my $processor = Text::Xslate->new(
         path   => $template_dir,
     );
-    my ($pages,$posts) = _post_helper($query, $tags, $query->{acls});
+    my @posts = _post_helper($query, $tags, $query->{acls});
 
     my $content = $processor->render('avatars.tx', {
-        users => $posts,
+        users => \@posts,
     });
 
     return [200,["Content-type: text/css\n"],[$content]];
@@ -639,8 +641,8 @@ Implements direct user profile view.
 
 sub users ($query, $render_cb) {
     push(@{$query->{acls}}, 'public');
-    my (undef,$posts) = _post_helper({ limit => 10000 }, ['about'], $query->{acls});
-    my @user = grep { $_->{user} eq $query->{username} } @$posts;
+    my @posts = _post_helper({ limit => 10000 }, ['about'], $query->{acls});
+    my @user = grep { $_->{user} eq $query->{username} } @posts;
     $query->{id} = $user[0]->{id};
     $query->{user_obj} = $user[0];
     return posts($query,$render_cb);
@@ -657,29 +659,29 @@ sub posts ($query, $render_cb) {
 
     push(@{$query->{acls}}, 'public');
     push(@{$query->{acls}}, 'unlisted') if $query->{id};
-    my ($pages,$posts);
+    my @posts;
 
     if ($query->{user_obj}) {
         #Optimize the /users/* route
-        $posts = [$query->{user_obj}];
+        @posts = ($query->{user_obj});
     } else {
-        ($pages,$posts) = _post_helper($query, $tags, $query->{acls});
+        @posts = _post_helper($query, $tags, $query->{acls});
     }
 
     #OK, so if we have a user as the ID we found, go grab the rest of their posts
-    if ($query->{id} && @$posts && grep { $_ eq 'about'} @{$posts->[0]->{tags}} ) {
-        my $user = shift(@$posts);
+    if ($query->{id} && @posts && grep { $_ eq 'about'} @{$posts[0]->{tags}} ) {
+        my $user = shift(@posts);
         my $id = delete $query->{id};
         $query->{author} = $user->{user};
-        ($pages, $posts) = _post_helper($query, [], $query->{acls});
-        @$posts = grep { $_->{id} ne $id } @$posts;
-        unshift @$posts, $user;
+        @posts = _post_helper($query, [], $query->{acls});
+        @posts = grep { $_->{id} ne $id } @posts;
+        unshift @posts, $user;
     }
 
-    return notfound($query, $render_cb) unless @$posts;
+    return notfound($query, $render_cb) unless @posts;
 
     my $fmt = $query->{format} || '';
-    return _rss($query,$posts) if $fmt eq 'rss';
+    return _rss($query,\@posts) if $fmt eq 'rss';
 
     my $processor = Text::Xslate->new(
         path   => $template_dir,
@@ -700,14 +702,16 @@ sub posts ($query, $render_cb) {
     my $styles = _build_themed_styles('posts.css');
 
     $query->{title} = @$tags ? "$query->{domain} : @$tags" : undef;
+    my $limit = int($query->{limit} || 25);
     my $content = $processor->render('posts.tx', {
         title     => $query->{title},
-        posts     => $posts,
+        posts     => \@posts,
         in_series => !!($query->{route} =~ m/\/series\/\d*$/),
         route     => $query->{route},
-        page      => int($query->{page} || 1),
-        limit     => int($query->{limit} || 25),
-        pages     => $pages,
+        limit     => $limit,
+        pages     => scalar(@posts) > $limit,
+        older     => $query->{older},
+        last      => $posts[-1]->{created},
         sizes     => [25,50,100],
         rss       => !$query->{id},
         tiled     => scalar(grep { $_ eq $query->{route} } qw{/files /audio /video /image /series /about}),
@@ -721,6 +725,7 @@ sub posts ($query, $render_cb) {
 sub _post_helper ($query, $tags, $acls) {
     state $data = Trog::Data->new($conf);
     return $data->get(
+        older   => $query->{older},
         page    => int($query->{page} || 1),
         limit   => int($query->{limit} || 25),
         tags    => $tags,
@@ -759,7 +764,7 @@ sub sitemap ($query, $render_cb) {
         # Return the index instead
         @to_map = ('static');
         my $data = Trog::Data->new($conf);
-        my $tot = $data->total_posts();
+        my $tot = $data->count();
         my $size = 50000;
         my $pages = int($tot / $size) + (($tot % $size) ? 1 : 0);
 
@@ -776,8 +781,7 @@ sub sitemap ($query, $render_cb) {
         # Return the map of the particular range of dynamic posts
         $query->{limit} = 50000;
         $query->{page} = $query->{map};
-        my (undef, $buf) = _post_helper($query, [], ['public']);
-        @to_map = @$buf;
+        @to_map = _post_helper($query, [], ['public']);
     }
 
     if ( $query->{xml} ) {

www/templates/paginator.tx

@@ -1,19 +1,21 @@
-:if ( $pages > 1 ) {
+:if ( $pages ) {
 <div>
 : } else {
 <div class="disabled">
 : }
-    <div style="float:left; margin-top:.5rem;"> Page <: $page :></div>
+    : if ( $older ) {
+    <div style="float:left; margin-top:.5rem;"> Posts older than <: $older :></div>
+    : }
     <div style="float:right;">
-        : if ( $page < $pages ) {
-        <a href="?page=<: $page + 1 :>&limit=<: $limit :>">Next</a>
-        : }
-        : if ( $page > 1 ) {
-        <a href="?page=<: $page - 1 :>&limit=<: $limit :>">Prev</a>
+        : if ( $older ) {
+        <a href="?older=<: $last :>&limit=<: $limit :>">Next</a>
         : }
+        <a href="?older=<: $older :>&limit=<: $limit :>">Prev</a>
         Size:
         <form style="display:inline;">
-            <input type="hidden" name="page"  value="<: $page :>" />
+            : if ( $older ) {
+            <input type="hidden" name="older"  value="<: $older :>" />
+            : }
             <select name="limit" class="coolbutton">
                 : for $sizes -> $size {
                 <option value="<: $size :>"><: $size :></option>