Git browser: Gemini/
This page presents code associated with the module/unit named above.
Gemini/gemini-planet-irc.sh
#!/bin/bash
yesterday=$(date --date yesterday +"%Y-%m-%d")
today=$(date --date today +"%Y-%m-%d")
echo "(ℹ) Planet Gemini updated. Latest complete date at gemini://gemini.techrights.org/planet/othercapsules/${yesterday}.gmi and so far today at gemini://gemini.techrights.org/planet/othercapsules/${today}.gmi with 3-day aggregate at gemini://gemini.techrights.org/planet/othercapsules/allinone.gmi" > /home/glr/ii-1.8/techrightslocal/irc.techrights.org/#techrights/in
Gemini/gemini-fetch-urls-from-rss.pl
#!/usr/bin/perl -T
# 2021-01-04
# updated 2021-02-12
# XML RSS and Atom feed web scraper,
# feed it URLs for feeds plus a date-time stamp
# entries will be parsed and can saved in a file
# local times will be converted to UTC
use utf8;
use Getopt::Std;
use Time::ParseDate;
use Time::Piece;
use XML::Feed;
use URI;
use LWP::UserAgent;
use HTTP::Response::Encoding;
use HTML::TreeBuilder::XPath;
binmode *STDOUT, ':utf8';
use English;
use strict;
use warnings;
our $VERBOSE = 0;
$OUTPUT_AUTOFLUSH=1;
# work-arounds for 'wide character' error from wrong UTF8
binmode(STDIN, ":encoding(utf8)");
binmode(STDOUT, ":encoding(utf8)");
our %opt;
getopts('ad:ho:uv', \%opt);
my $script = $0;
if (defined($opt{'h'})) {
&usage($script);
}
if (defined($opt{'v'})) {
$VERBOSE++;
}
my $output = '';
if (defined($opt{'o'})) {
# XXX needs proper sanity checking for path and filename at least
$output = $opt{'o'};
$output =~ s/[\0-\x1f]//g;
if ($output =~ /^([-\/\w\.]+)$/) {
$output = $1;
} else {
die("Bad path or file name: '$output'\n");
}
}
my $utc = 0; # treat input as a local time and convert to UTC
if (defined($opt{'u'})) {
$utc = 1; # treat input as UTC without conversion
}
# accept feed entries only after this date, default is yesterday
my $sdts;
if (defined($opt{'d'})) {
$sdts = parsedate($opt{'d'}, GMT=>$utc); # interpret input
} else {
$sdts = parsedate('yesterday'); # default to yesteday
}
my $t = Time::Piece->strptime($sdts, '%s'); # time in epoch seconds
print STDERR qq(S=$sdts\n)
if ($VERBOSE);
print STDERR qq(D=),$t->strftime("%a, %d %b %Y %H:%M:%S %Z"),qq(\n)
if ($VERBOSE);
# process feed URLs one at a time, keep track of how many there were
my $count = 0;
while (my $url = shift) {
next if ($url =~ /^\s*#/); # skip comments
print STDERR qq(\nU=$url\n)
if ($VERBOSE);
# get the feed and work on it
my $r = &get_feed($t,$url,$output);
if ($r) {
$count++;
} else {
print STDERR qq(Could not find feed at URL: "$url"\n);
}
}
# print qq(\n
\n\n) if ($count);
exit(1) unless ($count);
exit(0);
sub usage {
my ($script) = (@_);
$script =~ s/^.*\///;
print <parse(URI->new($uri)); };
if ($@) {
print STDERR $@,qq(\n);
print STDERR qq( Failed feed for '$uri'\n);
return(0);
}
# move past otherwise fatal failures in parsing titles, jump to next
my $feed_title;
eval { $feed_title = $feed->title; };
if ($@) {
print STDERR qq( Failed title for '$uri'\n);
return(0);
}
my $feed_modified = $feed->modified; # unsupported
my $feed_format = $feed->format;
print STDERR qq(\tT=$feed_title\n)
if ($VERBOSE);
print STDERR qq(\tF=$feed_format\n)
if ($VERBOSE);
# feed was acquired now go through and find the URLs it contains
&read_entries($t,$feed,$output);
return(1);
}
# find URLs within a feed
sub read_entries {
my ($t,$feed,$output) = (@_);
$t = parsedate($t);
# if there is an output file set it up for writing
my $out;
if($output) {
# choose append or overwrite mode for file
my $mode = '';
if (defined($opt{'a'})) {
$mode = '>>';
} else {
$mode = '>';
}
# open file for output in chosen mode and handling UTF-8
open($out, $mode, $output)
or die("Could not open '$output' for appending: $!\n");
binmode($out, ":encoding(utf8)");
}
my $count = 0;
foreach my $entry ($feed->entries) {
# entry time
my $ft = $entry->{entry}{pubDate}
|| $entry->issued
|| $entry->modified;
# entry time in seconds
my $et = parsedate($ft) || 0;
next unless($et =~ /^\d+$/ && $et >= $t );
# these links are sometimes redirections from proxies
my ($base, $content) = &fetch_page($entry->link)
or die("Missing content from '",$entry->link,"'\n");
next if ($base eq -1 || $content eq -1);
next if ($base =~ /^\d+/ && $base<0);
print STDERR qq(Fetched:),substr($base,0,30),qq(\n)
if ($VERBOSE);
my $uri = URI->new($base)
or warn("Bad address, '$base', could not form URI\n");
$uri->query(undef);
$uri->fragment(undef);
my $site = $uri->authority;
print STDERR qq(A=$site\n)
if ($VERBOSE);
my $o = $uri->canonical;
if ($o) {
$count++;
if($output) {
print $out $o,qq(\n);
} else {
print $o,qq(\n);
}
}
print STDERR qq(\t\t),$base,qq(\n)
if ($VERBOSE);
}
if($output) {
close $out;
}
return(1);
}
sub fetch_page {
my ($uri) = (@_);
# this part could just as well be replaced by wget or curl via system()
my $ua = LWP::UserAgent->new;
$ua->agent("TechRights-Gemini-Bot/0.1");
my $request = HTTP::Request->new(GET => $uri);
my $result = $ua->request($request);
if ($result->is_success) {
return($result->base, $result->decoded_content);
} else {
warn("Could not open '$uri' : ", $result->status_line, "\n");
return(-1,-1);
}
return(-1,-1);
}
Gemini/agate-tcpdump-logger.service
[Unit] Description=Logging for the Agate Gemini Server After=network.target After=agate.target [Service] User=root Type=simple Environment=STARTED_BY_SYSTEMD=true ExecStart=/usr/local/sbin/agate-tcpdump-logger.sh ExecReload=/bin/kill -HUP $MAINPID ExecStop=/bin/kill $MAINPID KillMode=control-group PrivateTmp=true Restart=on-failure Restart=5s [Install] WantedBy=multi-user.target Alias=gemini-logger.service
Gemini/gemini-get-and-convert-wiki-from-html.pl
#!/usr/bin/perl
# 2021-06-30
# scrape a designated wiki page and convert to gemtext
use utf8;
use Getopt::Std;
use File::Glob ':bsd_glob';
use HTML::TreeBuilder::XPath;
use HTML::Entities;
use URI;
binmode *STDIN, ':utf8';
# binmode *STDOUT, ':utf8';
use English;
use warnings;
use strict;
$OUTPUT_AUTOFLUSH = 1;
our %opt;
my $script = $0;
getopts('Chvw:', \%opt);
&usage($script) if ($opt{'h'});
# iterate through any parmeters passed on the command line,
# treat as file names and allow globbing
my @filenames;
while (my $file = shift) {
if ($file eq '-') {
# allow processing of STDIN
push(@filenames, '-');
if($opt{'v'}) { print qq(F= *STDIN\n); }
} else {
my @files = bsd_glob($file);
foreach my $f (@files) {
push(@filenames, $f);
if($opt{'v'}) { print qq(F=$f\n); }
}
}
}
my $outfile = $opt{'w'} || '';
chomp($outfile);
print "1: ", $outfile,qq(\n) if($opt{'v'});
# lll
( $outfile ) = ($outfile =~ m/^([\:\"\'\/\-\_\.\(\)\p{Word}\x7f-\xff]+)$/ );
print "2: ", $outfile,qq(\n) if($opt{'v'});
# quit if no files were listed
&usage($script) if($#filenames < 0);
# process each file, skipping turd files, hidden files, and temp files
while (my $infile = shift(@filenames)) {
next if ($infile =~ m/~$/);
next if ($infile =~ m/^\.\.?(?!\/)/);
next if ($infile =~ m/^#/);
my $result = &wiki_to_gemini($infile) || exit(1);
if ($outfile) {
my ($dir) = ($outfile =~ m{^(.*/)[^/]+$});
if (! -d $dir) {
die("Directory '$dir' does not exist.\n");
}
if (! -w $dir) {
die("Directory '$dir' is not writable.\n");
}
open(my $o, '>:utf8', $outfile)
or die("Could not open '$outfile' for writing: $!\n");
if ($opt{'v'}) { print qq(Writing to "$outfile"\n); }
print $o $result;
print $o qq(\n),qq(-)x10,qq(\n);
print $o qq(=>\t/\tTechrights\n);
print $o qq(➮ Sharing is caring. );
print $o qq(Content is available under CC-BY-SA.);
close($o);
} else {
print $result;
print qq(\n),qq(-)x10,qq(\n);
print qq(=>\t/\tTechrights\n);
print qq(➮ Sharing is caring. );
print qq(Content is available under CC-BY-SA.\n);
}
}
exit(0);
sub usage {
my ($script) = (@_);
$script =~ s/^.*\///g;
print qq(Usage: $script [-Chv] [-w file] file\n);
print <new;
$xhtml->implicit_tags(1);
$xhtml->no_space_compacting(0);
$xhtml->parse_file($input)
or die("Could not parse '$file' : $!\n");
my $result = '';
my %prefix = (
'h1' => "#\t● ",
'h2' => "##\t●● ",
'h3' => "###\t●●● ",
'h4' => "###\t●●●● ",
'h5' => "###\t●●●●● ",
'h6' => "###\t●●●●●● "
);
my @content = ();
for my $wiki ($xhtml->findnodes('//div[@id="bodyContent"]')) {
$wiki->detach();
push(@content, $wiki);
}
my $post = HTML::Element->new('div');
$post->push_content(@content);
if ($opt{'C'}) {
print qq(Deleting TOC\n) if($opt{'v'});
for my $toc ($post->findnodes('//table[@id="toc"]')) {
$toc->delete;
}
}
for my $script ($post->findnodes('//script')) {
$script->delete;
}
for my $div ($post->findnodes('//div/div/div')) {
$div->delete;
}
for my $table ($post->findnodes('//table[@id="toc"]')) {
$table->delete;
}
for my $table ($post->findnodes('//table[@class="wikitable"]')) {
$table->delete;
}
for my $a ($post->findnodes('//a[@name and not(@href)]')) {
$a->replace_with_content($a->as_text());
}
foreach my $hn (1 .. 5) {
# format headings
$hn = qq(h$hn);
for my $heading ($post->findnodes("//$hn")) {
my $h ="\n";
$h .= $prefix{$hn} if (defined($prefix{$hn}));
$h .= $heading->as_text."\n";
my $tmp = HTML::Element->new('~literal', 'text'=>$h);
$heading->replace_with($tmp);
}
}
for my $ul ($post->findnodes('.//ul')) {
for my $li ($ul->findnodes('.//li')) {
my $listitem = '';
if ($li->findnodes('text()')) {
$listitem = $li->as_text;
chomp($listitem);
}
my @anchors = ();
for my $a ($li->findnodes('.//a[@href and not(ancestor::pre)]')) {
my $href = $a->attr('href') || next;
my $uri = URI->new($href)->canonical || next;
my $text = $a->as_trimmed_text();
# normalize TR addresses
if(!$uri->scheme) {
$uri->scheme('http');
$uri->host('techrights.org');
} elsif($uri->scheme eq 'mailto') {
next;
} elsif($uri->host eq 'boycottnovell.com') {
$uri->host('techrights.org');
}
# mark external link or else convert TR links to Gemini
if ($uri->host ne 'techrights.org') {
$text = '↺ '.$text;
} elsif ($uri->host eq 'techrights.org' && ! $opt{'C'}) {
if($uri->path =~ m{\.pdf$}i) {
$text = '↺ '.$text;
} else {
$uri->host('gemini.techrights.org');
$uri->scheme('gemini');
unless ($uri->path =~ m/\/$/) {
my $p = $uri->path;
$uri->path($p.'/');
}
}
}
my $link = "\n=>\t".$uri->canonical."\t".$text;
push(@anchors, $link);
}
$listitem = $listitem.join("\n", @anchors).qq(\n);
my $tmp = HTML::Element->new('~literal', 'text'=>$listitem);
$li->replace_with($tmp);
}
}
for my $pp ($post->findnodes(".//p")) {
my @anchors=();
for my $a ($pp->findnodes('.//a[@href and not(ancestor::pre)]')) {
my $href = $a->attr('href');
print " H=$href\n" if ($opt{'v'});
next if ($href =~ m/^#/);
$href =~ s/\s/%20/g;
my $uri = URI->new($href)->canonical || next;
my $text = $a->as_trimmed_text();
# normalize TR addresses
if(!$uri->scheme) {
$uri->scheme('http');
$uri->host('techrights.org');
} elsif($uri->scheme eq 'mailto') {
next;
} elsif($uri->host eq 'boycottnovell.com') {
$uri->host('techrights.org');
}
# mark external link or else convert TR links to Gemini
if ($uri->host ne 'techrights.org') {
$text = '↺ '.$text;
} elsif ($uri->host eq 'techrights.org' && ! $opt{'C'}) {
if($uri->path =~ m{\.pdf$}) {
$text = '↺ '.$text;
} else {
$uri->host('gemini.techrights.org');
$uri->scheme('gemini');
unless ($uri->path =~ m/\/$/) {
my $p = $uri->path;
$uri->path($p.'/');
}
}
}
my $link = "\n=>\t".$uri->canonical."\t".$text;
push (@anchors, $link);
}
my $d;
if($#anchors >= 0) {
$d = $pp->as_text.qq(\n)
.join("\n", @anchors ).qq(\n);
} else {
$d = $pp->as_text.qq(\n);
}
my $tmp = HTML::Element->new('~literal', 'text' => $d);
$pp->replace_with($tmp);
}
for my $hr ($post->findnodes("//hr")) {
my $tmp = HTML::Element->new('~literal', 'text'=>"\n");
$hr->replace_with($tmp);
}
$result = $post->as_XML_indented;
$post->destroy;
$xhtml->destroy;
unless($is_stdin) {
close($input);
}
while ($result =~ s/<[^>]+>/ /g) { 1 }; # dead tags
while ($result =~ s/^\s+#/#/g) { 1 }; # fix headings
while ($result =~ s/^\s+$//) { 1 }; # trailing white space
while ($result =~ s/^\n\n\n/\n\n/gm) { 1 }; # vertical white space
$result = decode_entities($result);
return($result);
}
Gemini/gemini-log-journalctl.sh
#!/bin/sh
# 2022-01-18
# try to extract gemini log info from systemd's journalctl
# and convert it to gemtext
since=$(ps -p $(pgrep -o agate) --no-headers -o start)
d=$(date +"%F")
#--until "$(date -d "$d -1 day" +'%F 23:59')" \
# set -xv
journalctl -q -u agate \
--since "$(date -d "$d -1 day" +'%F 00:00')" \
| awk '
$7~/INFO/ && $11 ~/"gemini/ {
sub(/^"/, "", $11)
sub(/"$/, "", $11)
sub(/\/index.gmi$/, "/", $11)
sub(/\/feed$/, "/feed/", $11)
sub(/gemini:\/\/gemini.techrights.org:1965\//,
"gemini://gemini.techrights.org/", $11)
url[$11]++
c++
}
END {
print(c " Unique Requests\n");
for (u in url) {
printf("%s\t%4d\t%s\n", u,url[u],u)
}
}' \
| sort -k2,2nr -k3,3 \
| head -n 31 \
| awk -v s="$since" 'FNR==1 {
print("# Techrights Gemini Capsule Usage\n")
print("Running since " s "\n")
print("## Top accessed pages yesterday:\n")
}
$2 {
printf("=> %s\t%4d\t%s\n",$3,$2,$3)
}
END {
print("\n=> / Techrights")
printf("➮ Sharing is caring. ")
printf("Content is available under CC-BY-SA.\n")
}' \
> /home/gemini/gemini/stats/top-usage-yesterday.gmi
Gemini/gemini-main-index-template.sh
#!/bin/sh
# 2021-02-15
# a template for generating the main index.gmi file
PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin
# set -xv
d=$(date +'%F')
# get formatted dates
mm=$(date +"%Y/%m/index.gmi\t%B" -d "$d"); # this month
# my=$(date +"%Y/%m/index.gmi\t%B" -d "$d -1 month"); # last month
my=$(date --date="$(date -d $d +%Y-%m-15) -1 month" +"%Y/%m/index.gmi\t%B");
dy=$(date +"%Y/%m/%d" -d "$d -1 day"); # yesterday
dd=$(date +"%Y/%m/%d" -d "$d"); # today
mbn=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d"); # this month
mb1=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$(date -d $d +%Y-%m-15) -1 month");
mb2=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -2 month");
mb3=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -3 month");
mb4=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -4 month");
mb5=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -5 month");
mb6=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -6 month");
mb7=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -7 month");
mb8=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -8 month");
mb9=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -9 month");
mb10=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -10 month");
mb11=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -11 month");
mb12=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -12 month");
mb13=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -13 month");
mb14=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -14 month");
mb15=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -15 month");
mb16=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -16 month");
mb17=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -17 month");
mb18=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -18 month");
mb19=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -19 month");
mb20=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -20 month");
mb21=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -21 month");
mb22=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -22 month");
mb23=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -23 month");
mb24=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -24 month");
mb25=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -25 month");
mb26=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -26 month");
mb27=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -27 month");
mb28=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -28 month");
mb29=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -29 month");
mb30=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -30 month");
mb31=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -31 month");
mb32=$(date +"bulletins/%Y/%m/index.gmi\t%Y %B" -d "$d -32 month");
tmplinks=$(mktemp --suffix "-index-links-gemini") || exit 1
tmpindex=$(mktemp --suffix "-index-idx-gemini") || exit 1
trap "rm -f -- $tmplinks; rm -f -- $tmpindex;" EXIT
set -e
# find the index files for those dates and extract the links
# find /home/gemini/gemini \
# -type f \
# \( -path "*/$dy/index.gmi" -o -path "*/$dd/index.gmi" \) \
# -exec sed -nre '/Whole Month/,${/Whole Month/d;p}' {} \; \
# | sort -r \
# > $tmplinks
#
# find /home/gemini/gemini/202? \
# -mindepth 3 -maxdepth 3 -type f -name index.gmi -print \
# | sort -r \
# | head -n 3 \
# | xargs awk '$1=="=>"\
# &&$2~/^.[0-9]{4}.[0-9]{2}.[0-9]{2}.[[:alnum:]]{2}/' \
find /home/gemini/gemini/202? \
-mindepth 3 -maxdepth 3 -type f -name index.gmi -print \
| sort -r \
| head -n 3 \
| xargs perl -a -n -e \
'$F[1]=~m{/[0-9]{4}/[0-9]{2}/[0-9]{2}/\w+} && print' \
> $tmplinks
# cat -n $tmp
cat << EOF > $tmpindex
Welcome to Techrights, the Gemini Capsule! ⛬
We also have a Web proxy at http://gemini.techrights.org/
# Overview
=> /intro/ Introduction (ℹ)
=> /about/ About this capsule ⌂
=> /archives.gmi Capsule archives §
=> /irc.gmi Contact us (IRC) 📧
=> /logo/logo.png Site logo (if your Gemini client supports that)
# Articles from Techrights (GemText)
## Latest Articles in Techrights
$(cat $tmplinks)
• Please note the above might be slightly out of date, they are synched from HTTP every 3 hours ⟳ Next update at approximately $(TZ=UTC date -d "+3 hours" +"%H:%M UTC").
## Monthly Archives
=> $(echo $mm) 🕮
=> $(echo $my) 🕮
=> /archive/ Older (2006 to present) 📚
# Additional Sections
## Organised Information
=> /wiki/ Techrights wiki ▤
## Code
=> /git/ Techrights git G
## Videos
=> /latest-videos/ Latest Techrights videos ◉
=> /videos/ All Techrights videos ◉
## Capsule Stats
=> stats/index.gmi Statistics for this month 🗠
## Syndication
=> /feed.xml RSS/XML ♲
=> /planet/ Planet Gemini ♲
### Gemini Feeds (GemText)
=> /feed/ Feed as GemText ♲
=> /daily-feed/ Daily Feeds ♲
## IPFS (Decentralised)
=> /ipfs/ IPFS Archive 🗜
## Bulletins from Techrights (Plain Text)
=> $(echo -n $mbn), $(echo $mbn | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb1), $(echo $mb1 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb2), $(echo $mb2 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb3), $(echo $mb3 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb4), $(echo $mb4 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb5), $(echo $mb5 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb6), $(echo $mb6 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb7), $(echo $mb7 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb8), $(echo $mb8 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb9), $(echo $mb9 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb10), $(echo $mb10 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb11), $(echo $mb11 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb12), $(echo $mb12 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb13), $(echo $mb13 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb14), $(echo $mb14 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb15), $(echo $mb15 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb16), $(echo $mb16 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb17), $(echo $mb17 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb18), $(echo $mb18 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb19), $(echo $mb19 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb20), $(echo $mb20 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb21), $(echo $mb21 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb22), $(echo $mb22 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb23), $(echo $mb23 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb24), $(echo $mb24 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb25), $(echo $mb25 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb26), $(echo $mb26 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb27), $(echo $mb27 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb28), $(echo $mb28 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb29), $(echo $mb29 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb30), $(echo $mb30 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb31), $(echo $mb31 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
=> $(echo -n $mb32), $(echo $mb32 | cut -c1-18 \
| sed 's|bulletins|/home/gemini/gemini|' \
| xargs du -sh | cut -f1) 🖃
• The World Wide Web 🗺 always contains the latest bulletins
• Sizes at the top are for all bulletins (for a given month) combined
### ➮ Sharing is caring. Content is available under CC-BY-SA. ⟲
EOF
cat $tmpindex
rm -f -- $tmplinks
rm -f -- $tmpindex
trap - exit
exit
Gemini/gemini-inventory.pl
#!/usr/bin/perl
# 2021-02-22
# surveys the designated directory for articles
# groups them by day in daily summaries for each day, then
# creates a feed of recent articles
# creates another feed of recent daily summaries
use utf8;
use Getopt::Std;
use File::Glob ':bsd_glob';
use Path::Iterator::Rule; # libpath-iterator-rule-perl
use POSIX qw(strftime);
use Date::Calc qw(Add_Delta_Days Month_to_Text);
use strict;
use warnings;
our %opt;
getopts('A:v', \%opt);
print qq(Verbose mode\n) if ($opt{'v'});
# iterate through any parmeters passed on the command line,
# treat as file names and allow globbing
my @filenames;
while (my $file = shift) {
my @files = bsd_glob($file);
foreach my $f (@files) {
if (-d $f && -w $f) {
push(@filenames, $f);
} else {
warn("'$f' is not a directory and/or not writable.\n");
}
}
}
if ($#filenames < 0) {
print qq(Example:\n);
my ($script) = ($0 =~ m{([^/]+)$});
print qq($script /home/foobar/\n);
exit(1);
}
print qq(Paths:\n\t),join("\n\t",@filenames),qq(\n) if ($opt{'v'});
# prepare a search through the file system
my $rule = Path::Iterator::Rule->new;
$rule->name("*.gmi");
$rule->min_depth(4);
$rule->max_depth(6);
# $rule->contents_match(qr/^#+\s+/); # this rule does not match correctly
# find the sought after file names
my @files = $rule->all( @filenames );
print qq(Files:\n\t),join("\n\t",@files),qq(\n) if ($opt{'v'});
# collect titles and file names from the found issues/numbers and volumes
our %issue = ();
for my $f ( @files ) {
$f =~ s{//} {/}g;
print qq(Found: $f\n) if ($opt{'v'});
if (my ( $base, $year, $month, $day, $issue )
= ($f =~ m{^(.*)/(\d{4})/(\d{2})/(\d{2})/(.*)/(\w+\.gmi)$})) {
my $title = &process_file($f);
my $gindex = "/$year/$month/$day/index.gmi";
my $gfile = "/$year/$month/$day/$issue/index.gmi";
my $link = "=>\t$gfile\t$month/$day $title";
print " /$year/$month/$day/$issue/\n" if ($opt{'v'});
push (@{$issue{$gindex}{'links'}}, $link);
$issue{$gindex}{'base'} = $base;
}
}
&daily_summaries();
&monthly_summaries();
if($opt{'A'}) {
my ($archive) = ($opt{'A'} =~ /^([\w\/\_\-]+)$/);
if($archive) {
print qq(Archive = $archive\n) if ($opt{'v'});
&grand_archive($archive);
}
}
exit(0);
sub process_file {
my ($file) = (@_);
my $title = '';
open(my $article, '<:encoding(UTF-8)', $file)
or die("Could not open file '$file' for reading: $!\n");
while (my $line = <$article>) {
if ($line =~ m/^#+\s+(.*)$/) {
# skip plain dates
next if($line =~ m/^\#+\s+\S+\s+\d{2}\.\d{2}/);
chomp($line);
($title) = ($line=~m/^\#+\s+\S+\s+(.*)$/);
last;
}
}
close($article);
return($title||'');
}
sub daily_summaries {
# daily summaries
my @key = sort keys %issue;
for my $k (@key) {
my $base = $issue{$k}{'base'};
my $out;
open ($out, ">:utf8", "$base$k")
or warn("Could not open file '$base$k' for output: error: $!\n");
my ($title) = ($k =~ m/^(.*)index.gmi$/);
my ($month) = ($title =~ m/^\/(.*)\/\d+\/$/);
print $out qq(#\tTechRights\t$title\n\n);
print $out qq(=>\t/index.gmi\tTechrights\n);
print $out qq(=>\t../index.gmi\tThe Whole Month $month\n\n);
for my $link(@{$issue{$k}{'links'}}) {
print $out qq($link\n);
}
print $out qq(\n);
close($out);
}
# save the latest data as a sort of feed
my $k = $key[$#key];
my $base = $issue{$k}{'base'};
$base =~ s|\/$||;
my $feed = "$base/feed/";
unless (-e $feed) {
mkdir($feed,0755)
or die("Could not create directory '$feed': $!\n");
} elsif (! -d $feed) {
die("Not a directory: '$feed' \n");
} elsif (! -w $feed) {
die("Cannot write to directory '$feed'\n");
}
my $out;
open ($out, ">:utf8", $feed.'index.gmi')
or die("Could not open file '".$base.$feed.
"index.gmi' for output: error: $!\n");
#
my ($title) = ($k =~ m/^(.*)index.gmi$/);
my ($month) = ($title =~ m/^\/(.*)\/\d+\/$/);
# iso-8601 format
$title =~ s|^/||;
$title =~ s|/$||;
$title =~ s|/|-|g;
my $latest = $title;
print $out qq(#\tTechrights\n\n);
print $out qq(#\tDaily Feed of Recent Articles as of $title\n\n);
print $out qq(=>\t/index.gmi\tTechrights\n\n);
# add the current day's links
for my $link(@{$issue{$k}{'links'}}) {
$link =~ s|\b\d{2}/\d{2}\s+||;
$link =~ s|^(=>\s*\S+)(\s+)|$1$2$title |;
print $out qq($link\n);
}
print $out qq(\n);
# add the previous day's links too
$k = $key[$#key-1];
($title) = ($k =~ m/^(.*)index.gmi$/);
# iso-8601 format
$title =~ s|^/||;
$title =~ s|/$||;
$title =~ s|/|-|g;
for my $link(@{$issue{$k}{'links'}}) {
$link =~ s|\b\d{2}/\d{2}\s+||;
$link =~ s|^(=>\s*\S+)(\s+)|$1$2$title |;
print $out qq($link\n);
}
print $out qq(\n=>\t/$month/index.gmi\tThe Whole Month $month\n\n);
# or for GMT formatted appropriately for your locale:
my $updatednow = strftime "Last updated %F %H:%M UTC\n", gmtime;
print $out $updatednow;
close($out);
# feed of daily summaries
$feed = $base.'/daily-feed/index.gmi';
open ($out, ">:utf8", $feed)
or die("Daily-feed: Could not open file '".$feed
."' for output: error: $!\n");
print $out qq(#\tTechrights\n\n);
print $out qq(#\tFeed of Recent Daily Summaries\n\n);
print $out qq(=>\t/index.gmi\tTechrights\n\n);
my ($y, $m, $d) = split(/-/, $latest);
for my $i (0 .. 10) {
my ($dy, $dm, $dd) = Add_Delta_Days($y, $m, $d, -$i);
my $index = sprintf("/%04d/%02d/%02d/index.gmi", $dy,$dm,$dd);
if (-f $base.$index) {
printf $out ("=> %s\t%04d-%02d-%02d\n", $index,$dy,$dm,$dd);
}
}
print $out qq(\n).$updatednow;
close($out);
return(1);
}
sub monthly_summaries {
# monthly summaries
my %volume = ();
for my $k (sort keys %issue) {
my $base = $issue{$k}{'base'};
# K=/2021/01/01/index.gmi
my ($title) = ($k =~ m/^(\/\d{4}\/\d{2}\/)/);
my ($month) = ($title);
my ($year) = ($month =~ m/^(\d{4})\//);
next unless ($month);
$volume{$month}{'title'} = $title;
$volume{$month}{'base'} = $base;
for my $link(@{$issue{$k}{'links'}}) {
push(@{$volume{$month}{'links'}}, $link);
}
}
for my $m (sort keys %volume) {
my $title = $volume{$m}{'title'};
my $base = $volume{$m}{'base'};
my ($year,$month) = ( $title =~ m/^\/(\d{4})\/(\d{2})/);
my $out;
open ($out, ">:utf8", "$base$m"."index.gmi")
or warn("Could not open file '$base$m' for output: error: $!\n");
print $out qq(#\tTechrights\t$title\n\n);
print $out qq(=>\t/index.gmi\tTechrights\n\n\n);
# print $out qq(=>\t../index.gmi\tThe Whole Year $year\n\n);
my $count = 0;
for my $link (@{$volume{$m}{'links'}}) {
print $out qq($link\n);
$count++;
print $out qq(\n) unless ($count % 5);
print $out qq(\n) unless ($count % 10);
}
# append navigation for next and previous months, if they exits
my $next_month = $month+1>12 ? 1 : $month+1;
my $next_year = $next_month < $month ? $year+1 : $year;
my $prev_month = $month-1<1 ? 12 : $month-1;
my $prev_year = $prev_month > $month ? $year-1 : $year;
# next and previous keys to look for
my $next_key = sprintf("/%04d/%02d/", $next_year, $next_month);
my $prev_key = sprintf("/%04d/%02d/", $prev_year, $prev_month);
if(defined($volume{$next_key}) or defined($volume{$prev_key})){
print $out qq(\n),qq(-)x10,qq(\n);
if(defined($volume{$next_key})){
print $out qq(=>\t),$next_key,qq(index.gmi\tNext Month\n);
}
if(defined($volume{$prev_key})){
print $out qq(=>\t),$prev_key,qq(index.gmi\tPrevious Month\n);
}
}
close($out);
}
}
sub grand_archive {
my ($archive) = (@_);
my $latest_year=0;
my %volume = ();
for my $k (sort keys %issue) {
my ($year,$month) = ($k =~ m/^\/(\d{4})\/(\d{2})\//);
next unless ($month);
$volume{$year}{$month}++;
$latest_year=$year if ($year > $latest_year);
}
my $out;
my $f = "$archive/index.gmi";
unless(-e $archive) {
# ought to be made recursive
mkdir($archive,0755)
or die("Could not create directory '$archive': $!\n");
}
open ($out, ">:utf8", "$f")
or die("Could not open file '$f' for output: error: $!\n");
print $out qq(#\tArchive of All Articles as of $latest_year\n\n);
print $out qq(=>\t/\t ↩ back to Techrights (Main Index)\n);
for my $y (reverse sort keys %volume) {
print $out qq(\n# $y\n\n);
for my $m (reverse sort keys %{$volume{$y}}) {
my $month = Month_to_Text($m,1);
print $out qq(=> /$y/$m/\t$month $y\n);
}
}
close($out);
}
Gemini/gemini-git-update.sh.orig
#!/bin/sh
# 2021-09-13
# update read-only git archive
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# set -x
set -e
test -d /home/gemini/gemini/git
test -w /home/gemini/gemini/git
if test -d /home/gemini/gemini/git/tr-git; then
cd /home/gemini/gemini/git/tr-git
git fetch --all
git reset --hard
git pull
else
cd /home/gemini/gemini/git
git clone ssh://tr-git-read-only/home/git/tr-git/
git pull
fi
# reassemble pages
cd /home/gemini/gemini/git/tr-git # to be sure
# git pull
echo "# Latest Git Commits" > ~/gemini/git/change/index.gmi
echo "\n## Latest Change" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git show --format='--8<---[ %H ] | [ %aD ]----' * >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "## Two Changes Ago" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git show HEAD~2 --format='--8<---[ %H ] | [ %aD ]----' * >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "## Three Changes Ago" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git show HEAD~3 --format='--8<---[ %H ] | [ %aD ]----' * >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "## Four Changes Ago" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git show HEAD~4 --format='--8<---[ %H ] | [ %aD ]----' * >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "## Five Changes Ago" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git show HEAD~5 --format='--8<---[ %H ] | [ %aD ]----' * >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "\n# Files changed in the past week" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git diff --stat @{7.days.ago} >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "\n# Older Commits (Latest Shown First)" >> ~/gemini/git/change/index.gmi
echo "\n\`\`\`" >> ~/gemini/git/change/index.gmi
git log --all --decorate --oneline --graph >> ~/gemini/git/change/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/change/index.gmi
echo "=> / back to Techrights (Main Index)" >> ~/gemini/git/change/index.gmi
echo "# List of Files (Self-Hosted Git)" > ~/gemini/git/files/index.gmi
echo "\`\`\`" >> ~/gemini/git/files/index.gmi
git ls-tree --full-tree -r --name-only HEAD >> ~/gemini/git/files/index.gmi
echo "\`\`\`\n" >> ~/gemini/git/files/index.gmi
echo "=> / back to Techrights (Main Index)" >> ~/gemini/git/files/index.gmi
echo "\n# Git browser\n" > Desktop-Utils/index.gmi
echo "=> /git/ Back to index\n" >> Desktop-Utils/index.gmi
find Desktop-Utils/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> Desktop-Utils/index.gmi
echo "# Git browser"\n > Gemini/index.gmi
echo "=> /git/ Back to index\n" >> Gemini/index.gmi
find Gemini/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> Gemini/index.gmi
echo "# Git browser"\n > IPFS/index.gmi
echo "=> /git/ Back to index\n" >> IPFS/index.gmi
find IPFS/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n' \
>> IPFS/index.gmi
echo "# Git browser"\n > IRC/index.gmi
echo "=> /git/ Back to index\n" >> IRC/index.gmi
find IRC/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> IRC/index.gmi
echo "# Git browser"\n > Links/index.gmi
echo "=> /git/ Back to index\n" >> Links/index.gmi
find Links/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> Links/index.gmi
echo "# Git browser"\n > sbin/index.gmi
echo "=> /git/ Back to index\n" >> sbin/index.gmi
find sbin/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> sbin/index.gmi
echo "# Git browser"\n > Unit-files/index.gmi
echo "=> /git/ Back to index\n" >> Unit-files/index.gmi
find Unit-files/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> Unit-files/index.gmi
echo "# Git browser"\n > sbin-tm/index.gmi
echo "=> /git/ Back to index\n" >> sbin-tm/index.gmi
find sbin-tm/ -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> sbin-tm/index.gmi
exit 0
Gemini/gemini-get-cosmo.sh
#!/usr/bin/sh
# SHELL=/usr/bin/sh
# export PATH=$PATH:/home/gemini/bin
# openssl s_client -quiet -crlf \
# -servername skyjake.fi \
# -connect skyjake.fi:1965 \
# | awk '{ print "response: " $0 }'
#gemini://skyjake.fi/~Cosmos/
#openssl s_client -quiet -crlf \
#-servername skyjake.fi \
#-connect skyjake.fi:1965 \
#| awk '{ print "response: " $0 }'
#gemini://skyjake.fi/~Cosmos/
openssl s_client -quiet -crlf \
-servername skyjake.fi \
-connect skyjake.fi:1965 \
| awk '{ print "" $0 }' > /tmp/cosmos
gemini://skyjake.fi/~Cosmos/
Gemini/gemini-get-and-convert-wiki-get-rss.pl
#!/usr/bin/perl
# 2021-07-03
# read a hard-coded RSS feed and extract the URLs for changed
# wikipages, if they were changed on or after the designated date
use utf8;
use Getopt::Std;
use Time::ParseDate;
use Time::Piece;
use XML::Feed;
use URI;
use strict;
use warnings;
my $script = $0;
our $VERBOSE = 0;
# work-arounds for 'wide character' error from wrong UTF8
binmode(STDIN, ":encoding(utf8)");
binmode(STDOUT, ":encoding(utf8)");
our %opt;
getopts('d:huv', \%opt);
if (defined($opt{'h'})) {
&usage($script);
}
if (defined($opt{'v'})) {
$VERBOSE++;
}
my $url ="http://techrights.org/wiki/index.php?title=Special:RecentChanges&feed=rss";
my $utc = 0; # treat input as a local time and convert to UTC
if (defined($opt{'u'})) {
$utc = 1; # treat input as UTC without conversion
}
my $sdts;
if (defined($opt{'d'})) {
$sdts = parsedate($opt{'d'}, GMT=>$utc);
} else {
$sdts = parsedate('yesterday');
}
print STDERR qq(S=$sdts\n)
if ($VERBOSE);
my $t = Time::Piece->strptime($sdts, '%s');
print STDERR qq(D=),$t->strftime("%a, %d %b %Y %H:%M:%S %Z"),qq(\n)
if ($VERBOSE);
my @urls = &get_feed_urls($t, $url);
print join("\n", @urls),qq(\n);
exit(0);
sub usage {
my ($script) = (@_);
$script =~ s/^.*\///;
print <parse(URI->new($uri));
};
if ($@) {
print STDERR $@,qq(\n);
print STDERR qq( Failed feed for '$uri'\n);
return(0);
}
my $count = 0;
foreach my $entry ($feed->entries) {
# print STDERR Dumper($entry),qq(\n\n)
# if($VERBOSE);
# entry time
my $ft = $entry->{entry}{pubDate}
|| $entry->issued
|| $entry->modified;
# entry time in seconds
my $et = parsedate($ft) || 0;
next unless($et =~ /^\d+$/ && $et >= $t );
my $link = URI->new($entry->link);
my %keywords = $link->query_form();
my $new_uri = URI->new();
$new_uri->scheme($link->scheme);
$new_uri->host($link->host);
$new_uri->path($link->path);
$new_uri->query('title='.$keywords{'title'});
push(@urls, $new_uri->canonical) if (defined($link));
}
@urls = &unique(@urls);
return(@urls)
}
sub unique {
my (@urls) = (@_);
my %link;
foreach my $u (@urls) {
$link{$u}++;
}
my @links=();
foreach my $l (sort keys %link) {
push(@links, $l);
}
return(@links);
}
Gemini/tr-gemini-git-update.sh
#!/bin/sh
# 2021-09-13
# update read-only git archive
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# set -x
set -e
documentroot=/home/gemini/techrights.org
test -d ${documentroot}/git
test -w ${documentroot}/git
if test -d ${documentroot}/git/tr-git; then
cd ${documentroot}/git/tr-git/
git fetch --all
git reset --hard
git pull
else
cd ${documentroot}/git/
git clone /home/git/tr-git/
git pull
fi
# reassemble pages
cd ${documentroot}/git/tr-git # to be sure
# git pull
echo "# Latest Git Commits" > ${documentroot}/git/change/index.gmi
echo "\n## Latest Change" >> ${documentroot}/git/change/index.gmi
echo "\n\`\`\`" >> ${documentroot}/git/change/index.gmi
git show --format='--8<---[ %H ] | [ %aD ]----' * >> ${documentroot}/git/change/index.gmi
echo "\`\`\`\n" >> ${documentroot}/git/change/index.gmi
echo "## Two Changes Ago" >> ${documentroot}/git/change/index.gmi
c=1
while [ $c -lt 5 ]; do
echo "\n\`\`\`" \
>> ${documentroot}/git/change/index.gmi
git show HEAD~1 \
--format='--8<---[ %H ] | [ %aD ]----' * \
>> ${documentroot}/git/change/index.gmi
echo "\`\`\`\n" \
>> ${documentroot}/git/change/index.gmi
c=$(($c + 1))
done
echo "\n# Files changed in the past week" \
>> ${documentroot}/git/change/index.gmi
echo "\n\`\`\`" \
>> ${documentroot}/git/change/index.gmi
git diff --stat @{7.days.ago} \
>> ${documentroot}/git/change/index.gmi
echo "\`\`\`\n" \
>> ${documentroot}/git/change/index.gmi
echo "\n=> / back to Techrights (Main Index)" \
>> ${documentroot}/git/change/index.gmi
echo "\n# Older Commits (Latest Shown First)" \
> ${documentroot}/git/all_changes/index.gmi
echo "\n\`\`\`" \
>> ${documentroot}/git/all_changes/index.gmi
git log --all --decorate --stat --graph \
| grep -v 'Author:' \
>> ${documentroot}/git/all_changes/index.gmi
echo "\`\`\`\n" \
>> ${documentroot}/git/all_changes/index.gmi
echo "\n=> / back to Techrights (Main Index)" \
>> ${documentroot}/git/all_changes/index.gmi
echo "# List of Files (Self-Hosted Git)" \
> ${documentroot}/git/files/index.gmi
echo "\`\`\`" \
>> ${documentroot}/git/files/index.gmi
git ls-tree --full-tree -r --name-only HEAD \
>> ${documentroot}/git/files/index.gmi
echo "\`\`\`\n" \
>> ${documentroot}/git/files/index.gmi
echo "=> / back to Techrights (Main Index)" \
>> ${documentroot}/git/files/index.gmi
for d in */ ; do
echo "# Git browser: $d" > "$d"/index.gmi
echo "This page presents code associated with the module/unit named above." >> "$d"/index.gmi
echo "=> /git/all_changes/ Summary of changes" >> "$d"/index.gmi
echo "=> /git/ Back to Git index" >> "$d"/index.gmi
echo "=> /git/agplv3/ Licence (AGPLv3)" >> "$d"/index.gmi
find "$d" -type f -not -name index.gmi \
-printf '## %p\n\n```\n' \
-exec cat {} \; \
-printf '\n```\n'\
>> "$d"index.gmi
echo "=> / Back to main index" >> "$d"/index.gmi
test -d "$d"/changes/ || mkdir -p "$d"/changes/
echo "Latest changes in $d (latest first)" > "$d"/changes/index.gmi
echo "=> /git/ Back to Git index" >> "$d"/changes/index.gmi
git log "$d" \
| sed -e '/Author/d' \
-e 's/commit /# Commit identifier /' \
-e 's/Date:/##Time of change: /' \
>> "$d"/changes/index.gmi
echo "=> / Back to main index" >> "$d"/changes/index.gmi
done
exit 0
Gemini/monitor-gemini.sh
#!/bin/sh
PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin
amfora= '~/amfora_1.7.2_linux_64-bit'
while true;
do
$amfora gemini.techrights.org &
p=$!
sleep 120
$amfora simplynews.metalune.xyz/theguardian.com &
p=$(echo "$! $p")
sleep 120
$amfora gemini://gemini.bortzmeyer.org/software/lupa/stats.gmi &
p=$(echo "$! $p")
sleep 120
kill $p
done
exit 0
Gemini/gemini-make-feed.pl
#!/usr/bin/perl
# read TR's gemtext files and guess at building an RSS feed from them
use Getopt::Std;
use File::Find;
use Date::Parse;
use Date::Calc qw( Add_Delta_Days Gmtime );
use Time::Local;
use XML::Feed;
use DateTime;
use English;
use strict;
use warnings;
our %opt;
getopts('d:ho:r:v', \%opt);
my $script = $0;
$script =~ s|^.*\/||;
&usage($script) if ($opt{'h'});
my $path = shift;
if (! defined($path) || ! -d $path) {
&usage($script, 1);
}
our $date;
if ($opt{'d'}) {
$date = str2time($opt{'d'}, 'UTC') || &usage($script, 2);
} else {
print "Date missing. Assuming yesterday.\n";
my ($year,$month,$day) = Gmtime();
($year,$month,$day) = Add_Delta_Days($year,$month,$day, -1);
print "$year, $month, $day\n";
$month = $month - 1;
$year = $year - 1900;
$date = timegm( 0, 0, 0, $day, $month, $year)
}
if (! defined($opt{'r'}) && ! -d $opt{'r'}) {
&usage($script, 4);
}
my ($output);
if (defined($opt{'o'})) {
# XXX needs proper sanity checking for path and filename at least
$output = $opt{'o'};
$output =~ s/[\0-\x1f]//g;
if ($output =~ /^([-\/\w\.]+)$/) {
$output = $1;
} else {
die("Bad path or file name: '$output'\n");
}
} else {
$output = '/dev/stdout';
}
our @retrieved_files;
File::Find::find({wanted => \&wanted}, $path);
my @files = @retrieved_files;
if ($opt{'v'}) {
print "Files:\n", join("\n",@files),"\n";
}
my %metadata = &read_files(@files);
my $feed = XML::Feed->new('RSS');
$feed->title('Techrights Gemini Feed');
$feed->modified(DateTime->now(time_zone=> 'UTC'));
$feed->link('gemini://gemini.techrights.org/feed.rss');
$feed->language('en');
$feed->description('Techrights Gemini feed');
# $feed->ttl(1440);
for my $f (sort keys %metadata) {
my $r = "^$opt{'r'}";
my $p = $f;
my $u = "gemini://gemini.techrights.org/";
$p =~ s/$r/$u/;
$p =~ s|(?from_epoch(epoch=>$time, time_zone=>"UTC");
$title =~ s/^[#\s●]+//;
my $entry = XML::Feed::Entry->new();
$entry->link($p);
$entry->title($title);
$entry->modified($time);
$feed->add_entry($entry);
}
if ($output eq '/dev/stdout') {
print $feed->as_xml;
} elsif ($output) {
open(my $out, ">", $output)
or die("Could not open '$output' for writing: $!\n");
binmode($out, ':encoding(utf8)');
print $out $feed->as_xml;
close($out);
}
exit(0);
sub usage {
my ($script, $reason) = (@_);
if ($reason == 1 ) {
print "Missing path to GemText files\n";
} elsif ($reason == 2 ) {
print "Bad date or date format.\n";
} elsif ($reason == 3) {
print "Date missing\n";
} elsif ($reason == 4) {
print "Document root (base) missing\n";
print "use the -r option to set it\n";
}
print "\n";
print "Usage: \n";
print "$script [options] -r documentroot path\n";
print " -d starting date, defaults to yesterday\n";
print " -o path to output file, defaults to stdout\n";
print " -r document root for the Gemini server. required\n";
print " -v verbose output\n";
print " -h help - this output\n";
exit(0);
}
sub wanted {
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks);
# print "D=$File::Find::name\n";
if ((($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks) = lstat($_)) &&
-f $File::Find::name &&
m/\.gmi$/ &&
$mtime >= $date) {
push(@retrieved_files, $File::Find::name);
# print"$File::Find::name\n";
}
}
sub read_files {
my (@files) = (@files);
my %md =();
for my $file (@files) {
open(my $fh, "<", $file)
or die("Could not open '$file' : $!\n");
my $title = '';
my $count = 0;
while (my $line = <$fh>) {
chomp($line);
if ($line =~ m/^#/ && $count++) {
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,
$size,$atime,$mtime) = stat($fh);
$title = $line;
$md{$file}{'title'} = $title;
$md{$file}{'time'} = $mtime;
last;
}
}
close($fh);
}
return(%md);
}
Gemini/show-new-visitors-count.sh
#!/bin/sh
#
# Initial edits 2021-02-27
# Improvements for scaling 2021-09-09 (RS)
PATH=/usr/bin:/bin:/home/gemini/bin
capsule='/home/gemini'
RED='\033[0;31m' # Set some colours to help separate output visually
AMBER='\033[0;33m'
YELLOW='\033[1;33m'
GREEN='\033[0;32m'
BLUE='\033[0;34m'
NC='\033[0m' # No color
alertthreshold=1000 # How many requests per 10 minutes (by default) are considered 'too much'
dt=$(date +"%F %H:%M %Z") # now
d=$(date -d "$dt" +"%F") # today
dt=$(date -d "$dt") # reformat now
lockfile="/tmp/lockfile-$(basename $0)"
trap "rm -f -- $lockfile;" EXIT
if [ "${FLOCK}" != "$lockfile" ]; then
if flock -en "$lockfile" -c date; then
exec env FLOCK="$lockfile" flock -en "$lockfile" "$0" "$@"
else
echo "Already Locked. Aborting." >&2
exit 1
fi
else
echo Locking
fi
echo ' 🅶🅴🅼🅸🅽🅸 🅳🅳🅾🆂 🅿🆁🅾🆃🅴🅲🆃🅸🅾🅽' # Yup, just DDOS Protection (alert and response)
printf "${GREEN}"
systemctl status agate | grep Active
printf "${NC}\n"
printf "${AMBER}██████████████████████${NC} TOP REQUESTS ${AMBER}█████████████████████████████████████████▉▊▋${NC}\n"
set -e
# limit to just latest 1,000 in each file as logs have grown very large
tail -n1000 $capsule/logs/gemini-log-* \
| awk '$3 {a[$3]++} END{ for (b in a) {print a[b],b}}' OFS="\t" \
| sort -k1,1n -k2,2 | tail -n6
printf "${RED}██████████████████████${NC} LATEST 100 ${RED}███████████████████████████████████████████▉▊▋${NC}\n"
tail -n100 $capsule/logs/gemini-log-$d.log \
| awk '$3 {a[$3]++} END{ for (b in a) {print a[b],b}}' OFS="\t" \
| sort -k1,1n -k2,2 | tail -n4
TOTAL=$(wc -l < $capsule/logs/gemini-log-$d.log)
LASTTOTAL=$(cat $capsule/last-total.txt)
sum=$(( $TOTAL - $LASTTOTAL )) # Get the diff
if [ $sum -gt $alertthreshold ] # If too drastic an increase
then
printf "${RED}"
echo -n $sum
echo " new requests in 10 minutes. Check logs." # Verbose
printf "${NC}\n"
echo ''
echo " Consider running: sudo iptables -I INPUT -i wlan0 -s {OFFENDER_IP}/24 -j DROP" # Only suggest, do not run (yet)
echo ${TOTAL} > $capsule/last-total.txt # Update tally so as to not suppress the next run
printf "${RED}"
cat $capsule/logo.txt
printf "${NC}\n"
exit 2 # Sound the system bell
fi
echo ${TOTAL} > $capsule/last-total.txt
# FOR ALL DAYS:
# HOSTS=$(cat $capsule/logs/gemini-log-* | awk '$3 {a[$3]++} END{ for (b in a) {print a[b],b}}' OFS="\t" \
# | sort -k1,1n -k2,2 | wc -l)
# For current day only
HOSTS=$(cat $capsule/logs/gemini-log-$d.log \
| awk '$3 {a[$3]++} END{ for (b in a) {print a[b],b}}' OFS="\t" \
| sort -k1,1n -k2,2 | wc -l)
# HOSTS=off # turn off the above for extra speed
printf "${YELLOW}█████████████${NC} ${TOTAL} total requests and ${HOSTS} known hosts ${YELLOW}█████████████████▉▊▋${NC}\n"
echo -n $sum
echo " requests since last run (usually 10 minutes ago)."
echo -n 'Alert threshold (system bell): '
echo -n $alertthreshold
echo ' or higher'
# printf "${RED}____________________________________________________________${NC}\n"
echo
printf " ${GREEN}Latest 20 requests${NC} (updated <10 mins ago)\n\n"
tail -n20 $capsule/logs/gemini-log-$d.log \
| sed -E -e 's/[^ ]+ //; s/ / ⃪ /;'
echo _________________________________________________________
printf "${BLUE}"
cat $capsule/logo.txt
echo _________________________________________________________
# wc -l $capsule/logs/gemini-log-* | cut -b1-7,38-47
# wc -l $capsule/logs/gemini-log-* | cut -d/ -f1,5- | sed 's/ \/gemini-log//' | sed 's/.log//'| sed s/'-'/' reqs '/ | sed s/'2021-'/' | 2021-'/ | sed s/'-'/' \/ '/ | sed s/'-'/' \/ '/
wc -l $capsule/logs/gemini-log-* \
| sed -e 's#/.*-log-#reqs | #;
s#-# / #g;
s#\.log$##;' \
printf "${NC}\n"
Gemini=$(wc -l $capsule/logs/gemini-* | tail -n1)
Pages=$(find $capsule/gemini -name '*.gmi' | wc -l)
Gemactive=$(/usr/sbin/service agate status | grep Active)
echo "# Techrights Gemini Capsule Stats" > $capsule/gemini/stats/index.gmi
echo "## Gemini requests since start of month: $Gemini " >> $capsule/gemini/stats/index.gmi
echo "## Total pages in capsule: $Pages " >> $capsule/gemini/stats/index.gmi
echo "## Uptime (daemon): $Gemactive " >> $capsule/gemini/stats/index.gmi # create page
echo "## Top accessed pages yesterday (highest number first):\n" >> $capsule/gemini/stats/index.gmi
cat $capsule/logs/top-yesterday.log >> $capsule/gemini/stats/index.gmi
echo "## Detailed (number of pages, by date) \n • Please note that all logs are deleted at end of each month (completely, permanently)" >> $capsule/gemini/stats/index.gmi
echo "" >> $capsule/gemini/stats/index.gmi
echo " Page requests ── Date" >> $capsule/gemini/stats/index.gmi
echo " ↧ ────────────── ↧" >> $capsule/gemini/stats/index.gmi
echo "┌────────────────────────────────┐" >> $capsule/gemini/stats/index.gmi
# wc -l $capsule/logs/gemini-log-* | cut -b1-8,39-47 >> $capsule/gemini/stats/index.gmi # send to capsule too
# wc -l $capsule/logs/gemini-log-* | cut -d/ -f0,5- | sed 's/ \/gemini-log//' | sed 's/.log//'| sed s/'-'/' reqs '/ | sed s/'2021-'/' | 2021-'/ | sed s/'-'/' \/ '/ | sed s/'-'/' \/ '/ >> $capsule/gemini/stats/index.gmi
wc -l $capsule/logs/gemini-log-* \
| sed -e 's#/.*-log-#reqs | #; s#-# / #g; s#\.log$##;' \
>> $capsule/gemini/stats/index.gmi
echo " ──────────────────────────────── " >> $capsule/gemini/stats/index.gmi
echo '## Daily Stats Archive' >> $capsule/gemini/stats/index.gmi
# Each year another block will need to be added below for the past years
y=$(date -d "$d -1 year" +"%Y")
echo "### $y Archive" >> $capsule/gemini/stats/index.gmi
echo "=> $y.gmi All days ($y)" >> $capsule/gemini/stats/index.gmi
echo "# $y Stats Archive" > $capsule/gemini/stats/$y.gmi
find /home/gemini/gemini/stats/ \
-type f \
-name "stats-$y*.gmi" \
-exec basename {} \; \
| sort \
| sed -r -e 's/([0-9]{4}-[0-9]{2}-[0-9]{2})\.gmi/\1.gmi \1/; s/^/=> /' \
-e '/01-01.*/i ### January ㋀' \
-e '/02-01.*/i ### February ㋁' \
-e '/03-01.*/i ### March ㋂' \
-e '/04-01.*/i ### April ㋃' \
-e '/05-01.*/i ### May ㋄' \
-e '/06-01.*/i ### June ㋅' \
-e '/07-01.*/i ### July ㋆' \
-e '/08-01.*/i ### August ㋇' \
-e '/09-01.*/i ### September ㋈' \
-e '/10-01.*/i ### October ㋉' \
-e '/11-01.*/i ### November ㋊' \
-e '/12-01.*/i ### December ㋋' \
>> $capsule/gemini/stats/$y.gmi
y=$(date -d "$d -2 year" +"%Y")
echo "### $y Archive" >> $capsule/gemini/stats/index.gmi
echo "=> $y.gmi All days ($y)" >> $capsule/gemini/stats/index.gmi
echo "# $y Stats Archive" > $capsule/gemini/stats/$y.gmi
find /home/gemini/gemini/stats/ \
-type f \
-name "stats-$y*.gmi" \
-exec basename {} \; \
| sort \
| sed -r -e 's/([0-9]{4}-[0-9]{2}-[0-9]{2})\.gmi/\1.gmi \1/; s/^/=> /' \
-e '/01-01.*/i ### January ㋀' \
-e '/02-01.*/i ### February ㋁' \
-e '/03-01.*/i ### March ㋂' \
-e '/04-01.*/i ### April ㋃' \
-e '/05-01.*/i ### May ㋄' \
-e '/06-01.*/i ### June ㋅' \
-e '/07-01.*/i ### July ㋆' \
-e '/08-01.*/i ### August ㋇' \
-e '/09-01.*/i ### September ㋈' \
-e '/10-01.*/i ### October ㋉' \
-e '/11-01.*/i ### November ㋊' \
-e '/12-01.*/i ### December ㋋' \
>> $capsule/gemini/stats/$y.gmi
echo '### This Year' >> $capsule/gemini/stats/index.gmi
y=$(date -d "$d" +"%Y")
find /home/gemini/gemini/stats/ \
-type f \
-name "stats-$y*.gmi" \
-exec basename {} \; \
| sort \
| sed -r -e 's/([0-9]{4}-[0-9]{2}-[0-9]{2})\.gmi/\1.gmi \1/; s/^/=> /' \
-e '/01-01.*/i ### January ㋀' \
-e '/02-01.*/i ### February ㋁' \
-e '/03-01.*/i ### March ㋂' \
-e '/04-01.*/i ### April ㋃' \
-e '/05-01.*/i ### May ㋄' \
-e '/06-01.*/i ### June ㋅' \
-e '/07-01.*/i ### July ㋆' \
-e '/08-01.*/i ### August ㋇' \
-e '/09-01.*/i ### September ㋈' \
-e '/10-01.*/i ### October ㋉' \
-e '/11-01.*/i ### November ㋊' \
-e '/12-01.*/i ### December ㋋' \
>> $capsule/gemini/stats/index.gmi
echo '• No personally-identifying data is in these pages. The raw logs get deleted.' >> $capsule/gemini/stats/index.gmi
echo " ──────────────────────────────── \n \n=> / back to Techrights (Main Index)" >> $capsule/gemini/stats/index.gmi
echo -n "🅶🅴🅼🅸🅽🅸 Page last updated at " >> $capsule/gemini/stats/index.gmi
echo $dt >> $capsule/gemini/stats/index.gmi
wget --quiet --tries=22 --retry-on-http-error=500,503 \
-O $capsule/gemini/chat/irc.gmi http://techrights.org/irc/log.gmi \
&& sleep 6 \
&& mv --backup -f \
$capsule/gemini/chat/irc.gmi $capsule/gemini/chat/index.gmi \
|| gemini-ping-roy.sh bright
# cat $capsule/techrights-logo-colour.txt
echo Unlocking
exit 0
Gemini/archives.gmi
Welcome to Techrights Archives, static section of the capsule => / Back to index # Techrights Archives ## Popular Sections => /epo/ EPO Articles ▤ => /linuxfoundation/ Linux Foundation ▤ => /gatesfoundation/ Gates Foundation ▤ => /wiki/ Techrights wiki ▤ => /chat/ Latest chat logs for #techrights (updated every 10 minutes) ▤ # Full Archives => /archive/ Monthly Archives of Articles => /videos/ Techrights videos ◉ => stats/index.gmi Statistics for this month 🗠 => /git/ Techrights git G ## Techrights IRC Archives => irc/index.gmi ㏒ - IRC logs for #techrights => social/index.gmi ㏒ - IRC logs for #boycottnovell-social => techbytes/index.gmi ㏒ - IRC logs for #techbytes => boycottnovell/index.gmi ㏒ - IRC logs for #boycottnovell ## Techrights Site Archives => /ipfs/ IPFS Archive 🗜 => /bulletins/ Techrights Bulletins 🖃 • Plain Text Bulletins are included, but may be hard to get over IPFS due to their size • The World Wide Web 🗺 site always contains the latest Techrights bulletins ## Techrights Syndication => /feed.xml RSS/XML ♲ => /planet/ Planet Gemini ♲ => /feed/ Feed as GemText ♲ => /daily-feed/ Daily Feeds ♲ => /logo/logo.png Site logo (if your Gemini client supports that) ### ➮ Sharing is caring. Content is available under CC-BY-SA. ⟲
Gemini/gemini-get-and-convert-wiki.pl
#!/usr/bin/perl -T
# 2021-07-05
# main script to launch two helper scripts:
# a) gemini-get-and-convert-wiki-get-rss.pl - read URLs from RSS
# b) gemini-get-and-convert-wiki-from-html.pl - convert from specific HTML to Gemini
# then feed the output from a into b and save as a file
use Time::ParseDate;
use Date::Calc qw(Localtime);
use URI;
use File::Path qw(make_path);
use Getopt::Std;
use URI::Escape;
use utf8;
use warnings;
use strict;
our %opt;
getopts('d:hUv', \%opt);
my $date = $opt{'d'} || 'now -2 days';
$date = &iso_8601_date($date);
die() unless($date);
my $destination = '/home/gemini/gemini/wiki';
$ENV{'PATH'} = '/home/gemini/bin:/usr/local/bin:/usr/bin:/bin';
my @urls = &read_rss_feed($date);
foreach my $url (@urls) {
my $uri = URI->new($url)->canonical;
my %query = $uri->query_form();
my $title = $query{'title'};
next if ($title =~ m/\.\./);
my $outdir = join('/',$destination,$title);
print qq(outdir = $outdir\n) if $opt{'v'};
&convert_html_to_gemini($uri, $outdir);
}
&write_index unless($opt{'U'});
exit(0);
sub iso_8601_date {
my ($date) = (@_);
my $seconds = parsedate($date);
my ($year,$month,$day) = Localtime($seconds);
$date = sprintf("%04d-%02d-%02d", $year,$month,$day);
return($date);
}
sub read_rss_feed {
my ($date) = (@_);
my $feed_source = "gemini-get-and-convert-wiki-get-rss.pl";
next unless($date =~ m/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/);
print qq(Date = $date\n) if ($opt{'v'});
open(my $fh, "-|", $feed_source, "-d", $date)
or die("Could not open feed source \"$feed_source\": $!\n");
my @urls;
while (my $url = <$fh>) {
chomp($url);
print qq(URL = $url\n) if ($opt{'v'});
push(@urls, $url);
}
close($fh);
return(@urls);
}
sub convert_html_to_gemini {
my ($uri, $outdir) = (@_);
unless (-e $outdir) {
make_path($outdir, {mode=>0755});
}
unless (-w $outdir) {
print STDERR qq(Cannot write to "$outdir"\n);
return(0);
}
($uri) = uri_unescape($uri);
my $outfile = join('/', $outdir, 'index.gmi');
# ($outfile) =~ ($outfile =~ m/^([^\x00-\x20]+)$/g);
print qq(\tO = $outfile\n) if ($opt{'v'});
my @doc = ();
$uri = uri_escape($uri,"\x20\x22\x27\x60");
($uri) = ($uri =~ m/^([^\x00-\x20]+)$/);
open(my $html, '-|', "curl", "-s", "$uri")
or die("Cannot open 'curl'\n");
while(my $line = <$html>) {
push(@doc, $line);
}
close($html);
print qq(\tOO = $outfile\n) if($opt{'v'});
open($html,'|-',"gemini-get-and-convert-wiki-from-html.pl",
"-w", "$outfile", "-")
or die("Cannot open converter\n");
foreach my $line (@doc) {
print $html $line;
}
close($html);
return(1);
}
sub write_index {
my $path = '/home/gemini/gemini/wiki';
my $base = '/wiki';
opendir(my $dir, $path)
or die("Could not read '$path': $!\n");
my @dirs=();
while (readdir($dir)) {
my $d = $_;
next unless(-d "$path/$d");
next if ($d =~ m/^\.+/);
push(@dirs, $_);
}
closedir($dir);
open(my $o, ">", "$path/index.gmi")
or die("Could not open \"$path/index.gmi\" for writing: $!\n");
print $o qq(# Techrights Wiki mirror\n\n);
print $o qq(=>\tgemini://gemini.techrights.org/\tTechrights\n\n);
foreach my $d (sort @dirs) {
my $dd = $d;
$dd =~ s/_/ /g;
print $o qq(=>\t$base/$d/\t$dd\n);
}
print $o qq(\n### ➮ Sharing is caring. );
print $o qq(Content is available under CC-BY-SA. ⟲ \n);
close($o);
}
Gemini/export-stats.sh
#/bin/sh
# 2021-09-15
# runs as 'pi'
PATH=/usr/bin:/bin
awk '
$5~/^agate/ {
if($11~/favicon\.txt/) {
next;
}
sub(/^"/,"",$11);
sub(/"$/,"",$11);
sub(/index\.gmi$/,"",$11);
sub("^gemini://gemini.techrights.org:1965/",
"gemini://gemini.techrights.org/",$11);
if($11!~/\/$/) {
$11=$11 "/";
}
if($11=="/") {
$11="gemini://gemini.techrights.org/";
}
a[$11]++
c++
}
END {
print(c " Unique Requests\n");
for(u in a) {
print(a[u],u)
}
}' OFS="\t" /var/log/syslog.1 \
| sort -k1,1nr -k2,2 \
| head -n 31 \
| awk ' NR==2 {
print("\nRequests\tPage")
}
NR>1 {
print("=> ", $2, $0 " ⇛")
}
NR<2 {
print;
}
' OFS="\t" \
> /home/gemini/logs/top-yesterday.log
echo -n 'Accessed over HTTP/S gateway: '>> /home/gemini/logs/top-yesterday.log
wc -l /var/log/nginx/access.log.1 | cut -f 1 -d ' ' >> /home/gemini/logs/top-yesterday.log
exit 0
Gemini/tr-gemini-latest-videos.pl
#!/usr/bin/perl
# 2021-09-17
# update latest video gallery page
use HTML::TreeBuilder::XPath;
use File::Basename;
use JSON;
print "# Latest Techrights Videos\n\n";
print "Most recent shown first (the list below is limited to past 7 days)\n\n";
my %videos = &xhtml_video_links('-');
my %metadata = &fetch_metadata(%videos);
foreach my $key (sort {$a<=>$b} keys %metadata) {
print "=> ";
print $metadata{$key}{'link'}," ";
print "↺ ";
printf ("%2d ",$key);
if (exists($metadata{$key}{'title'})) {
print $metadata{$key}{'title'};
if(exists($metadata{$key}{'date'})){
print " (",$metadata{$key}{'date'},")";
}
} else {
print $metadata{$key}{'file'};
}
print "\n";
}
print "\n=> /videos/ Show videos archive\n";
print "=> / Back to homepage\n";
exit(0);
sub xhtml_video_links{
my ($file)= (@_);
my $is_stdin = 0;
my $input;
if ($file eq '-') {
$input = *STDIN;
$is_stdin++;
} else {
# force input to be read in as UTF-8
open ($input, "<:utf8", $file)
or die("Could not open file '$file' : error: $!\n");
}
my $xhtml = HTML::TreeBuilder::XPath->new;
$xhtml->implicit_tags(1);
$xhtml->no_space_compacting(0);
$xhtml->parse_file($input)
or die("Could not parse '$file' : $!\n");
my %videos = ();
my $listpos = 0; # reset position at 0
for my $l ($xhtml->findnodes_as_strings('//a[@target="video"]/@href') ) {
my $link = qq(http://techrights.org/videos/).$l;
my $vid = basename($link);
$listpos++;
$videos{$listpos}{'link'} = $link;
$videos{$listpos}{'file'} = $vid;
}
$xhtml->destroy;
unless($is_stdin) {
close($input);
}
return(%videos);
}
sub fetch_metadata() {
my (%metadata) = (@_);
foreach my $listpos (keys %metadata) {
my @cmd = ("ffprobe",
"-v", "panic", "-of", "json", "-show_format",
$metadata{$listpos}{'link'});
open(my $json, "-|", @cmd)
or die("Could not open pipe '@cmd' : $!\m");
my @text = ();
while (my $j = <$json>) {
push (@text, $j);
}
close($json);
my $t = join("", @text);
my $d = decode_json(join("",@text));
# print Dumper($d),"\n";
if ($d->{'format'}->{'tags'}->{'title'}) {
$metadata{$listpos}{'title'} = $d->{'format'}->{'tags'}->{'title'};
}
if ($d->{'format'}->{'tags'}->{'DATE'}) {
$metadata{$listpos}{'date'} = $d->{'format'}->{'tags'}->{'DATE'};
}
# print $metadata{$listpos}{'link'},"\n";
# print $metadata{$listpos}{'file'},"\n\n";
}
return(%metadata);
}
Gemini/gemini-cron-updater.sh
#!/bin/sh
# 2021-02-15 Update TR Gemini site from TR HTTP RSS
#
# fetch the TR RSS feed with the first script
# (gemini-fetch-urls-from-rss.pl)
# fetch the linked web pages with the second,
# (gemini-fetch-web-page.pl)
# which internally calls a third script
# (gemini-parse-html-to-gemini.pl)
# that third script converts the page from HTML to Gemini
# and places the result in the file system hierarchy
# then if successful, rebuild the Gemini indexes using a fourth script
# (gemini-main-index-template.sh)
PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin
# override environment variables set incorrectly by the SSH client
export LANG="en_GB.UTF-8"
export LC_ALL="en_GB.UTF-8"
export LC_TIME="en_GB.UTF-8"
export LC_MONETARY="en_GB.UTF-8"
export LC_ADDRESS="en_GB.UTF-8"
export LC_TELEPHONE="en_GB.UTF-8"
export LC_NAME="en_GB.UTF-8"
export LC_MEASUREMENT="en_GB.UTF-8"
export LC_IDENTIFICATION="en_GB.UTF-8"
export LC_NUMERIC="en_GB.UTF-8"
export LC_PAPER="en_GB.UTF-8"
lockfile="/tmp/lockfile-$(basename $0)"
if [ "${FLOCK}" != "$lockfile" ]; then
if flock -en "$lockfile" -c date; then
exec env FLOCK="$lockfile" flock -en "$lockfile" \
-c "$0" "$@"
else
echo "Already Locked. Aborting." >&2
exit 4
fi
else
true
# echo Locking
fi
sudo /usr/local/sbin/tc-shaper-v2021-Nov.sh
tmpfeeds=$(mktemp --suffix "-cron-gemini") || exit 1
trap "rm -f -- $lockfile; rm -f -- $tmplinks; rm -f -- $tmpindex;" EXIT
set -e
yyyy=$(date +"%Y");
count=5
while ! timeout 200 gemini-fetch-urls-from-rss.pl http://techrights.org/feed/ \
> ${tmpfeeds}
do
count=$((${count}-1))
if [ ${count} -eq 0 ]
then
exit 2
fi
sleep 5
done
for u in $(grep http ${tmpfeeds});
do
count=5
while ! timeout 200 gemini-fetch-web-page.pl \
-g -p -b /home/gemini/gemini/ ${u}
do
count=$((${count}-1))
if [ ${count} -eq 0 ]
then
exit 3
fi
sleep 5
done
done
timeout 600 gemini-inventory.pl -A /home/gemini/gemini/archive/ \
/home/gemini/gemini/ \
&& gemini-main-index-template.sh > /home/gemini/gemini/index.gmi
# gemini-inventory.pl /home/gemini/gemini/2021 works, too, but
# truncates the archive, if used
gemini-bulletin-irc-update.sh
# let's wait for the site to update its index (as it does
# every 5 minutes) and then retrieve the list in case
# a new video was posted (note that the file of the video
# gets uploaded well before a post gets published, so sleep()
# might be spurious)
sleep 25
gemini-latest-videos.sh
rm -f -- ${lockfile}
rm -f -- ${tmpfeeds}
trap - exit
sudo /usr/local/sbin/tc-shaper-v2.sh
exit 0
Gemini/gemini-index-years-only.sh
#!/bin/sh
# 2021-09013
PATH=/usr/bin:/bin
cd /home/gemini/gemini
find 2??? \
-maxdepth 0 \
-type d \
-exec sh -c "(echo '# Techrights {}\n'; ls -d {}/?? | \
sed -re 's|^.*/||; s|(.*)|=> \1/ \1|') \
> {}/index.gmi" \;
exit 0
Gemini/gemini-capcom.gmi
=> gemini://sbg.one/gemfeed/atom.xml => gemini://lab6.com/atom.xml => gemini://basnja.ru/atom.xml => gemini://makeworld.gq/gemlog/atom.xml => gemini://si3t.ch/log/atom.xml => gemini://alexschroeder.ch:1965/do/blog/atom => gemini://capsule.theparanoidtimes.org/posts/atom.xml => gemini://ainent.xyz/gemlog/index.gmi => gemini://l-ipa.net/atom.xml => gemini://gemini.susa.net/atom.xml => gemini://tilde.team/~sumpygump/gemlog/atom.xml => gemini://heathens.club/~palm93/the_woods/ => gemini://hoseki.iensu.me/atom.xml => gemini://six10.pw => gemini://9til.de/users/~julienxx/atom.xml => gemini://thfr.info/index.gmi => gemini://pachapunk.space/atom.xml => gemini://etam-software.eu/ => gemini://freeshell.de/gemlog/atom.xml => gemini://envs.net/~vee/pikkulog/atom.xml => gemini://bobignou.red/atom.xml => gemini://acidic.website/musings/atom.xml => gemini://edwardtefft.com/atom.xml => gemini://mizik.eu/feed.xml => gemini://miso.town/atom.xml => gemini://jdj.golf/gemlog/atom.xml => gemini://cap.swan.quest/p/atom.xml => gemini://amami.city/~xdefrag/atom.xml => gemini://knijn.srht.site/posts/atom.xml => gemini://gemlog.blue/users/cariboudatascience/ => gemini://aprates.dev/log/atom.xml => gemini://misterbanal.net/atom.xml => gemini://skyjake.fi/gemlog/atom.xml => gemini://idf.looting.uk/capslog/atom.xml => gemini://dira.cc/gemlog/atom.xml => gemini://ebc.li/atom.xml => gemini://tanelorn.city/~bouncepaw/gemlog/feed.rss => gemini://benj.smol.pub/atom.xml => gemini://smcttl.flounder.online/gemlog/atom.xml => gemini://gemini.circumlunar.space/users/solderpunk/pikkulog/atom.xml => gemini://reisub.nsupdate.info/test/atom.xml => gemini://rosenzweig.io/gemlog/atom.xml => gemini://xhrpb.com/feed.xml => gemini://drewdevault.com/feed.xml => gemini://envs.net/~vee/gemlog/atom.xml => gemini://sylvaindurand.org/rss.xml => gemini://tanso.net/atom.xml => gemini://random-projects.net/feed.atom => gemini://www.underworld.fr/atom.xml => gemini://sylvaindurand.org/feed.xml => gemini://cee64.us/feed.gmi => gemini://tracciabi.li/whiterabbit/index.gmi => gemini://gluonspace.com/gemlog/atom.xml => gemini://rtr.kalayaan.xyz/blog.gmi/atom.xml => gemini://roughcustomers.com => gemini://makeworld.space/gemlog/atom.xml => gemini://taoetc.org/ => gemini://rootkey.co.uk/posts/atom.xml => gemini://gemini.ctrl-c.club/~nehrman/gemlog/atom.xml => gemini://shit.cx/atom.xml => gemini://deblan.io/feed.xml => gemini://c3d2.de/news-atom.xml => gemini://celehner.com/feed.xml => gemini://ondollo.com/~/zert:7C:8E:F5/atom.gem => gemini://gemini.circumlunar.space/users/acdw/atom.xml => gemini://laika.lk/atom.xml => gemini://725.be/index.gmi => gemini://www.demorrow.net/atom.xml => gemini://republic.circumlunar.space/users/crdpa/blog/atom.xml => gemini://muscar.eu/feeds/feed.atom.xml => gemini://gemini.circumlunar.space/~adiabatic/atom.xml => gemini://beyondneolithic.life/atom.xml => gemini://phreedom.club/~tolstoevsky/glog/atom.xml => gemini://gemini.mat.services/atom.xml => gemini://gemini.bbbhltz.space/gemlog/atom.xml => gemini://tilde.club/~lewiscowper/gemlog/atom.xml => gemini://posixcafe.org/blogs/feed.atom => gemini://shuhao.srht.site/posts/atom.xml => gemini://gnuser.land/gemlog/atom.xml => gemini://gemlog.blue/users/NetCandide/ => gemini://spool-five.com/gemlog/index.gmi => gemini://c3po.aljadra.xyz/blog.gmi => gemini://talon.computer/log/atom.xml => gemini://nader.pm/atom.xml => gemini://c3po.aljadra.xyz/atom.xml => gemini://gemini.ucant.org/gemlog/atom.xml => gemini://sylvaindurand.org/atom.xml => gemini://low-key.me/gemlog/atom.xml => gemini://gem.johanbove.info/gemlog/atom.xml => gemini://tilde.pink/~emily/atom.xml => gemini://taoetc.org/feed.gmi => gemini://gemini.circumlunar.space/users/alchemist/gemlog/atom.xml => gemini://warmedal.se/~bjorn/atom.xml => gemini://ruario.flounder.online/gemlog/atom.xml => gemini://blog.schmidhuberj.de/atom.xml => gemini://tilde.team/~ivanruvalcaba/glog/atom.xml => gemini://tilde.team/~easeout/glog/atom.xml => gemini://koyu.space/blu256/atom.xml => gemini://gem.vectorpoem.com/projects_log.gmi => gemini://perso.pw/blog/rss.xml
Gemini/watch-for-heavy-users.sh
#!/bin/bash
HEIGHT=15
WIDTH=40
CHOICE_HEIGHT=4
BACKTITLE="DDOS observer"
TITLE="Gemini @ Techrights"
MENU="Choose refresh/recheck interval:"
OPTIONS=(1 "Every 1 minute"
2 "Every 10 minutes (recommended, sound/beep suppressed)"
3 "Every 30 minutes"
)
CHOICE=$(dialog --clear \
--backtitle "$BACKTITLE" \
--title "$TITLE" \
--menu "$MENU" \
$HEIGHT $WIDTH $CHOICE_HEIGHT \
"${OPTIONS[@]}" \
2>&1 >/dev/tty)
clear
case $CHOICE in
1)
echo "You chose Option 1"
watch -b -c -n 60 ~gemini/bin/show-new-visitors-count.sh ;;
2)
echo "You chose Option 2"
watch -c -n 600 ~gemini/bin/show-new-visitors-count.sh ;;
3)
echo "You chose Option 3"
watch -b -c -n 1800 ~gemini/bin/show-new-visitors-count.sh ;;
esac
# watch -b -c -n 600 ~gemini/bin/show-new-visitors-count.sh
Gemini/gemini-feeds.gmi
=> gemini://mozz.us/journal/atom.xml => gemini://midnight.pub/feed.xml => gemini://gemini.circumlunar.space/users/solderpunk/gemlog/atom.xml => gemini://gemini.circumlunar.space/~solderpunk/hitenheroes/posts/atom.xml => gemini://gemini.circumlunar.space/~solderpunk/gemlog/atom.xml => gemini://gemini.circumlunar.space/~solderpunk/cornedbeef/atom.xml => gemini://republic.circumlunar.space/users/flexibeast/gemlog/atom.xml => gemini://gemini.circumlunar.space/~ew/feed.xml => gemini://drewdevault.com/feed.xml => gemini://treeprophet.flounder.online/gemlog/atom.xml => gemini://mothbaby.flounder.online/gemlog/atom.xml => gemini://cetacean.club/journal/atom.xml => gemini://hannuhartikainen.fi/twinlog/atom.xml => gemini://gemini.jayeless.net/gemlog/atom.xml => gemini://idiomdrottning.org/atom.xml => gemini://going-flying.com/~mernisse/atom.xml => gemini://breadpunk.club/~bagel/atom.xml => gemini://nytpu.com/flightlog/atom.xml => gemini://nytpu.com/gemlog/atom.xml => gemini://avalos.me/gemlog/atom.xml => gemini://rawtext.club/~mieum/prose/atom.xml => gemini://breadpunk.club/~snickerdoodles/recipes/atom.xml => gemini://rawtext.club/~mieum/relog/atom.xml => gemini://rawtext.club/~mieum/poems/atom.xml => gemini://gemini.sensorstation.co/atom.xml => gemini://envs.net/~kyrenaios/atom.xml => gemini://tilde.team/~easeout/glog/atom.xml => gemini://gemini.conman.org/boston.atom => gemini://carcosa.net/send-the-nukes/atom.xml => gemini://carcosa.net/journal/atom.xml => gemini://yam655.com/recent-feed.xml => gemini://envs.net/~negatethis/atom.xml => gemini://upyum.com/journal/feed.atom => gemini://1436.ninja/gemlog/gemlog.atom => gemini://80h.dev/glog/atom.xml => gemini://80h.dev/~int/glog/atom.xml => gemini://9til.de/users/~julienxx/atom.xml => gemini://caracolito.mooo.com/~sejo/atom.xml => gemini://tilde.team/~emilis/atom.xml => gemini://republic.circumlunar.space/users/joneworlds/atom.xml => gemini://rosenzweig.io/gemlog/atom.xml => gemini://breadpunk.club/~bakersdozen/gemlog/atom.xml => gemini://calcuode.com/gemlog/atom.xml => gemini://salejandro.me/gemlog/atom.xml => gemini://gemini.circumlunar.space/users/solderpunk/pikkulog/atom.xml => gemini://warmedal.se/~bjorn/atom.xml => gemini://gemini.circumlunar.space/~solderpunk/pikkulog/atom.xml => gemini://hedy.flounder.online/gemlog/atom.xml => gemini://antiphasis.flounder.online/gemlog/atom.xml => gemini://asp.flounder.online/gemlog/atom.xml => gemini://jfh.me/atom.xml => gemini://ecs.d2evs.net/feed.xml => gemini://chartjunk.flounder.online/gemlog/atom.xml => gemini://enteka.flounder.online/gemlog/atom.xml => gemini://gem.acdw.net/do/atom => gemini://gem.acdw.net/do/rss => gemini://gem.acdw.net/do/all/atom => gemini://going-flying.com/thoughts/atom.xml => gemini://tilde.team/~ivanruvalcaba/glog/atom.xml => gemini://gemini.ctrl-c.club/~nehrman/gemlog/atom.xml => gemini://gemini.circumlunar.space/users/adiabatic/atom.xml => gemini://rawtext.club/~tolstoevsky/glog/atom.xml => gemini://rawtext.club/~verkaro/glog//atom.xml => gemini://tilde.club/~lewiscowper/gemlog/atom.xml => gemini://gemini.circumlunar.space/users/parker/archives/atom.xml => gemini://gemini.circumlunar.space/users/parker/gacme/atom.xml => gemini://gemini.circumlunar.space/~swiftmandolin/gemlog/atom.xml => gemini://tilde.team/~sumpygump/gemlog/atom.xml => gemini://l-3.space/atom.xml => gemini://republic.circumlunar.space/users/itsdave/atom.xml => gemini://republic.circumlunar.space/~itsdave/noc/atom.xml => gemini://republic.circumlunar.space/~itsdave/atom.xml => gemini://republic.circumlunar.space/users/luminar/atom_feed.xml => gemini://rwv.io/src/atom.xml => gemini://rwv.io/atom.xml => gemini://m040601.flounder.online/gemlog/atom.xml => gemini://testuser.flounder.online/atom.xml => gemini://testuser.flounder.online/myfeed.xml => gemini://thelovebug.flounder.online/gemlog/atom.xml => gemini://zettelkastensystem.flounder.online/gemlog/atom.xml => gemini://gem.acdw.net/code/do/atom => gemini://gem.acdw.net/code/do/rss => gemini://gem.acdw.net/food/do/atom => gemini://gem.acdw.net/food/do/rss => gemini://gem.acdw.net/life/do/atom => gemini://gem.acdw.net/life/do/rss => gemini://gem.acdw.net/made/do/atom => gemini://gem.acdw.net/made/do/rss => gemini://gem.acdw.net/pub/do/atom => gemini://gem.acdw.net/pub/do/rss => gemini://gem.acdw.net/text/do/atom => gemini://gem.acdw.net/text/do/rss => gemini://gemini.susa.net/atom.xml => gemini://pulham.info/atom.xml => gemini://mrnd.xyz/log/atom.xml => gemini://her.esy.fun/gem-atom.xml => gemini://celehner.com/feed.xml => gemini://gemini.circumlunar.space/~solderpunk/3albums/atom.xml => gemini://otrn.org/atom.xml => gemini://acidic.website/musings/atom.xml => gemini://gemini.thegonz.net/glog/atom.xml => gemini://samsai.eu/gemlog/atom.xml => gemini://apintandaparma.club/~ajc/log/atom.xml => gemini://sunshinegardens.org/~xj9/feed.atom => gemini://envs.net/~lrb/gemlog/atom.xml => gemini://perplexing.space/atom.xml => gemini://gemini.circumlunar.space/~adiabatic/atom.xml => gemini://apintandaparma.club/~ajft/phlog/atom.xml => gemini://gempaper.strangled.net/notes/atom.xml => gemini://republic.circumlunar.space/users/dbane/gemlog/atom.xml => gemini://tilde.pink/~lucymoth/gemlog/atom.xml => gemini://aoalmeida.com/feed.xml => gemini://shit.cx/atom.xml => gemini://tilde.team/~emilis/feed.xml => gemini://gsthnz.com/posts/atom.xml => gemini://skyjake.fi/gemlog/atom.xml => gemini://emii.gay/en/emi-overshares/atom.xml => gemini://republic.circumlunar.space/~luminar/atom_feed.xml => gemini://drsudo.com/gemlog/atom.xml => gemini://gemini.ucant.org/gemlog/atom.xml => gemini://gemini.iosa.it/gemlog/atom.xml => gemini://freeside.wntrmute.net/log/index.rss => gemini://orx57.flounder.online/gemlog/atom.xml => gemini://rawtext.club/~mieum/music/atom.xml => gemini://rawtext.club/~ecliptik/_posts/feed.xml => gemini://mnmnm.flounder.online/gemlog/atom.xml => gemini://kujiu.org/blog/atom.xml => gemini://nerv-project.eu/blog/atom.xml => gemini://gmi.karl.berlin/atom.xml => gemini://tposs.flounder.online/gemlog/atom.xml => gemini://gemini.lottalinuxlinks.com/posts/feed.xml => gemini://nader.pm/atom.xml => gemini://dsfadsfgafgf.flounder.online/gemlog/atom.xml => gemini://space.fqserv.eu/gemlog/atom.xml => gemini://moddedbear.xyz/logs/atom.xml => gemini://lightspeed.flounder.online/gemlog/atom.xml => gemini://shtirlic.flounder.online/gemlog/atom.xml => gemini://makeworld.space/users/~atyrfingerprints/gemlog/atom.xml => gemini://makeworld.space/gemlog/atom.xml => gemini://phreedom.club//atom.xml => gemini://jochen.flounder.online/gemlog/atom.xml => gemini://lycopersica.flounder.online/gemlog/atom.xml => gemini://rasputin.selfip.net/atom.xml => gemini://space.fqserv.eu/tanka/atom.xml => gemini://thmslld.flounder.online/gemlog/atom.xml => gemini://moribundo.flounder.online/gemlog/atom.xml => gemini://gemini.cyberbot.space/gemlog/atom.xml => gemini://breadpunk.club/~proof/atom.xml => gemini://breadpunk.club/~snickerdoodles/mouthfuls/atom.xml => gemini://riqtare.flounder.online/gemlog/atom.xml => gemini://przemek.flounder.online/gemlog/atom.xml => gemini://szczezuja.flounder.online/gemlog/atom.xml => gemini://votih.flounder.online/gemlog/atom.xml => gemini://cosmic.voyage/atom.xml => gemini://random-projects.net/feed.atom => gemini://muppet.flounder.online/gemlog/atom.xml => gemini://scifirenegade.flounder.online/gemlog/atom.xml => gemini://etam-software.eu/blog/feed.xml => gemini://adnano.co/atom.xml => gemini://edwardtefft.com/atom.xml => gemini://t-900.flounder.online/gemlog/atom.xml => gemini://sunbance.flounder.online/gemlog/atom.xml => gemini://jlk.flounder.online/gemlog/atom.xml => gemini://capsule.usebox.net/gemlog/atom.xml => gemini://filter.id.au/gemlog/atom.xml => gemini://tilde.pink/~emily/atom.xml => gemini://senders.io/feed/atom.xml => gemini://senders.io/gemlog/feed/atom.xml => gemini://1436.ninja/phlog.atom => gemini://flume.space/atom.xml => gemini://wirthslaw.flounder.online/gemlog/atom.xml => gemini://pwshnotes.flounder.online/gemlog/atom.xml => gemini://bonehead.flounder.online/gemlog/atom.xml => gemini://luke.flounder.online/gemlog/atom.xml => gemini://gemini.grappling.ca/gemlog/atom.xml => gemini://fungihirn.flounder.online/gemlog/atom.xml => gemini://mederle.de/feed-de => gemini://mederle.de/feed-en => gemini://heathens.club/~palm93/atom.xml => gemini://tilde.team/~ivanruvalcaba/blog/atom.xml => gemini://tilde.club/~liamvhogan/blog/atom.xml => gemini://soviet.circumlunar.space/markov/schismatrix-notes/atom.xml => gemini://phreedom.club/~tolstoevsky/glog/atom.xml => gemini://tilde.pink/~monerulo/atom.xml => gemini://posixcafe.org/blogs/feed.atom => gemini://posixcafe.org/vocaloid/feed.atom => gemini://gemini.mcgillij.dev/atom.xml => gemini://gemini.bunburya.eu/gemlog/posts/atom.xml => gemini://ew.srht.site/feed.xml => gemini://dantes.flounder.online/gemlog/atom.xml => gemini://my32.flounder.online/gemlog/atom.xml => gemini://si3t.ch/log/atom.xml => gemini://gemini.circumlunar.space/~alchemist/gemlog/atom.xml => gemini://blog.snowfrost.garden/atom.xml => gemini://www.underworld.fr/blog/atom.xml => gemini://envs.net/~vee/gemlog/atom.xml => gemini://gem.rmgr.dev/blog/atom.xml => gemini://inconsistentuniverse.space/atom.xml => gemini://s.tymo.name/log/atom.xml => gemini://envs.net/~vee/pikkulog/atom.xml => gemini://gem.pwarren.id.au/gemlog/atom.xml => gemini://rawtext.club/~mieum/dallok/atom.xml => gemini://qd.discordian.de/entries/atom.xml => gemini://tilde.pink/~hektor/atom.xml => gemini://caolan.uk/atom.xml => gemini://gemini.astropirados.space/gemlog/atom.xml => gemini://www.underworld.fr/atom.xml => gemini://g.dumke.me/log/atom.xml => gemini://rek2.hispagatos.org/posts/atom.xml => gemini://gemini.ctrl-c.club/~axeflayer/atom.xml => gemini://bi-rabittoh.flounder.online/gemlog/atom.xml => gemini://axiomatika.flounder.online/gemlog/atom.xml => gemini://cdaniels.net/feed_http.rss => gemini://cdaniels.net/feed_http.atom => gemini://cdaniels.net/feed_gmi.rss => gemini://cdaniels.net/feed_gmi.atom => gemini://buetow.org/gemfeed/atom.xml => gemini://gemini.cyberbot.space/smolzine/atom.xml => gemini://0gitnick.flounder.online/atom.xml => gemini://nicksphere.com/atom.xml => gemini://crystal.flounder.online/gemlog/atom.xml => gemini://ilogique.flounder.online/gemlog/atom.xml => gemini://renedarioherrera.flounder.online/gemlog/atom.xml => gemini://thek3nger.flounder.online/gemlog/atom.xml => gemini://breadpunk.club/~snickerdoodles/meta/atom.xml => gemini://figbert.com/log/atom.xml => gemini://szczezuja.space/gemlog/atom.xml => gemini://birabittoh.smol.pub/atom.xml => gemini://ocramoi.flounder.online/gemlog/atom.xml => gemini://corstar.flounder.online/gemlog/atom.xml => gemini://freeshell.de/gemlog/atom.xml => gemini://degrowther.smol.pub/atom.xml => gemini://starbreaker.smol.pub/atom.xml => gemini://edim.flounder.online/gemlog/atom.xml => gemini://gnuser.land/gemlog/atom.xml => gemini://thwidge.flounder.online/gemlog/atom.xml => gemini://ghostglyph.flounder.online/gemlog/atom.xml => gemini://tilde.cafe/~hedy/feed.xml => gemini://gemini.marmaladefoo.com/cgi-bin/atom-feed.cgi?lukee => gemini://mieum.smol.pub/atom.xml => gemini://hedy.tilde.cafe/feed.xml => gemini://gemini.spam.works/users/emery/atom.feed => gemini://memex.smol.pub/atom.xml => gemini://compudanzas.net/atom.xml => gemini://euromancer.flounder.online/gemlog/atom.xml => gemini://0x80.org/gemlog/atom.xml => gemini://hedy.smol.pub/atom.xml => gemini://reisub.nsupdate.info/fabianbonetti/atom.xml => gemini://cobaltblue.flounder.online/gemlog/atom.xml => gemini://erifnella.flounder.online/gemlog/atom.xml => gemini://tilde.team/~supernova/gemlog/atom.xml => gemini://rawtext.club/~mieum/atom.xml => gemini://blog.locrian.zone/atom.xml => gemini://snonux.de/gemfeed/atom.xml => gemini://hugeping.ru/atom.xml => gemini://melyanna.flounder.online/gemlog/atom.xml => gemini://miguelmurca.flounder.online/gemlog/atom.xml => gemini://november.smol.pub/atom.xml => gemini://cowface.online/gemlog/atom.xml => gemini://fawn.garden/log/atom.xml => gemini://tilde.team/~g1n/blog/feed.rss => gemini://subphase.xyz/users/flow/gemlog/atom.xml => gemini://twh.flounder.online/gemlog/atom.xml => gemini://jayeless.flounder.online/gemlog/atom.xml => gemini://devinprater.flounder.online/gemlog/atom.xml => gemini://alsd.eu/it/feed.xml => gemini://alsd.eu/en/feed.xml => gemini://bleyble.com/users/quokka/atom.xml => gemini://dimension.sh/~nikdoof/logs/atom.xml => gemini://tilde.team/~blumenkiste/atom.xml => gemini://thelambdalab.xyz/atom.xml => gemini://matthewhall.xyz/gemlog/atom.xml => gemini://karmanyaah.malhotra.cc/gemlog/atom.xml => gemini://gmi.bacardi55.io/gemlog/atom.xml => gemini://gem.keazilla.net/atom.xml => gemini://gemini.alexandrevicente.net/atom.xml => gemini://dira.cc/gemlog/atom.xml => gemini://aperalesf.flounder.online/gemlog/atom.xml => gemini://nickj.flounder.online/gemlog/atom.xml => gemini://reisub.nsupdate.info/planetalibre/atom.xml => gemini://dustypenguin.flounder.online/gemlog/atom.xml => gemini://g.mikf.pl/gemlog/atom.xml => gemini://mkf.flounder.online/gemlog/atom.xml => gemini://monika.flounder.online/gemlog/atom.xml => gemini://kota.nz/atom.xml => gemini://mischk.flounder.online/gemlog/atom.xml => gemini://epi.benthic.zone/atom.xml => gemini://byzoni.org/gemlog/atom.xml => gemini://orkney.flounder.online/gemlog/atom.xml => gemini://isoraqathedh.pollux.casa/atom.xml => gemini://hugeping.ru/micro/atom.xml => gemini://danihopera.flounder.online/gemlog/atom.xml => gemini://niedzwiedzinski.cyou/feed.xml => gemini://hugeping.tk/micro/atom.xml => gemini://miso.town/atom.xml => gemini://m15o.smol.pub/atom.xml => gemini://ichthys.tilde.cafe/atom.xml => gemini://marginalia.nu/log/feed.xml => gemini://envs.net/~anonyth/atom.xml => gemini://sev.flounder.online/gemlog/atom.xml => gemini://ataraxia.flounder.online/gemlog/atom.xml => gemini://privacy.flounder.online/gemlog/atom.xml => gemini://gemini.panda-roux.dev/log/feed.xml => gemini://foobucket.xyz/gemlog/atom.xml => gemini://warmedal.se/~antenna/atom.xml => gemini://maren.flounder.online/gemlog/atom.xml => gemini://eaplmx.smol.pub/atom.xml => gemini://envs.net/~armen/gemlog/atom.xml => gemini://mieum.smol.space/atom.xml => gemini://phreedom.club/atom.xml => gemini://blog.datapulp.de/atom.xml => gemini://datapulp.smol.pub/atom.xml => gemini://aprates.dev/log/atom.xml => gemini://alexwennerberg.com/gemlog/atom.xml => gemini://alex.flounder.online/gemlog/atom.xml => gemini://text.eapl.mx/atom.xml => gemini://jsreed5.org/feeds/log.xml => gemini://jsreed5.org/feeds/math.xml => gemini://jsreed5.org/feeds/changelog.xml => gemini://soviet.circumlunar.space/wholesomedonut/gemlog/atom.xml => gemini://drafts.cyclexo.com/gemlog/atom.xml => gemini://s0.is/projects/atom.xml => gemini://tilde.cafe/~ichthys/atom.xml => gemini://iceworks.cc/z/atom.xml => gemini://vy.binarylab.eu/gemlog/atom.xml => gemini://officialdonut.smol.pub/atom.xml => gemini://viridian.flounder.online/gemlog/atom.xml => gemini://hajime.4teri.de/dotfiles/atom.xml => gemini://gemini.ctrl-c.club/~nristen/gemlog/atom.xml => gemini://aprates.dev/pt-br/log/atom.xml => gemini://stacksmith.flounder.online/gemlog/atom.xml => gemini://blekksprut.net/nikki/atom.xml => gemini://breadpunk.club/~snickerdoodles/atom.xml => gemini://josias.dev/gemlog/feed.xml => gemini://szczezuja.space/git/scripts/atom.xml => gemini://szczezuja.space/git/gmidiff/atom.xml => gemini://tilde.team/~tomasino/atom.xml => gemini://beyondneolithic.life/atom.xml => gemini://lertsenem.com/atom.xml => gemini://tilde.town/~nihilazo/atom.xml => gemini://moonrockcenter.flounder.online/gemlog/atom.xml => gemini://gemini.bvnf.space/blog.rss => gemini://ichi.smol.pub/atom.xml => gemini://sequel.space/bubble/K1gTCL2PVZJN7ZZ1VPFp98/atom.xml => gemini://sequel.space/bubble/ViPx8vf3YSCUisLSKW1rzT/atom.xml => gemini://sequel.space/bubble/PVnyeuvYXKkA18UFFC6ZZy/atom.xml => gemini://sequel.space/bubble/GgZW37jjdLk4JPSeie7tCW/atom.xml => gemini://sequel.space/bubble/CWYmF993GKX94aSwufXU5y/atom.xml => gemini://sequel.space/bubble/Xg2Zdu3nYdSDMKD6qBiYM3/atom.xml => gemini://sequel.space/bubble/9NmeAJSkwGTHtoS3Xr1rw2/atom.xml => gemini://sequel.space/bubble/H6ognLcBjqQSa6MhbuDGKN/atom.xml => gemini://sequel.space/bubble/GLTatsSbmzzejurBzJ5dHS/atom.xml => gemini://sequel.space/bubble/7h3uWLuZbgzvB99d4rdnPC/atom.xml => gemini://sequel.space/bubble/TTToQ7FsbQGqw23SYzQ5un/atom.xml => gemini://sequel.space/bubble/HngiQfQ9cAQChi5ghRTcAF/atom.xml => gemini://sequel.space/bubble/9octt9FwLbbVumUkpjUgZP/atom.xml => gemini://sequel.space/bubble/Xm395CWNAT8fbUAt8dmVQr/atom.xml => gemini://sequel.space/bubble/Azs3YBJ78ytjuFgzf5KyHV/atom.xml => gemini://sequel.space/bubble/5Z3SGnuJEJND3tsR9iBrgM/atom.xml => gemini://sequel.space/bubble/JUQwKFrE8qKWH265BdYp3U/atom.xml => gemini://sequel.space/bubble/6EyhnusCdAV5MseoLFU9o1/atom.xml => gemini://sequel.space/bubble/JguD5Ws1tDvvfJrbKN7TGW/atom.xml => gemini://sequel.space/bubble/NXP4oChaEPJpd5rSkRPsk1/atom.xml => gemini://sequel.space/bubble/RnA2gV1Lcj6nFvdHRxRoYp/atom.xml => gemini://sbg.one/SandboxGeneral/atom.xml => gemini://gemini.circumlunar.space:1965/users/adiabatic/atom.xml => gemini://auragem.space/devlog/atom.xml => gemini://ax.flounder.online/gemlog/atom.xml => gemini://galaxyhub.uk/articles/atom.xml => gemini://gemini.circumlunar.space:1965/~solderpunk/gemlog/atom.xml => gemini://gem.acdw.net:1965/do/atom => gemini://gem.acdw.net:1965/do/rss => gemini://gem.acdw.net:1965/do/all/atom => gemini://gem.acdw.net:1965/code/do/atom => gemini://gem.acdw.net:1965/code/do/rss => gemini://gem.acdw.net:1965/food/do/atom => gemini://gem.acdw.net:1965/food/do/rss => gemini://gem.acdw.net:1965/life/do/atom => gemini://gem.acdw.net:1965/life/do/rss => gemini://gem.acdw.net:1965/made/do/atom => gemini://gem.acdw.net:1965/made/do/rss => gemini://gem.acdw.net:1965/pub/do/atom => gemini://gem.acdw.net:1965/pub/do/rss => gemini://gem.acdw.net:1965/text/do/atom => gemini://gem.acdw.net:1965/text/do/rss => gemini://gemini.circumlunar.space:1965/users/parker/archives/atom.xml => gemini://gemini.circumlunar.space:1965/users/parker/gacme/atom.xml => gemini://gemini.circumlunar.space:1965/users/rbasile/blog/feed.atom => gemini://gemini.circumlunar.space:1965/users/solderpunk/gemlog/atom.xml => gemini://gemini.circumlunar.space:1965/users/solderpunk/pikkulog/atom.xml => gemini://mizik.eu/feed.xml => gemini://rawtext.club:1965/~mieum/atom.xml => gemini://rawtext.club:1965/~mieum/poems/atom.xml => gemini://rawtext.club:1965/~mieum/prose/atom.xml => gemini://rawtext.club:1965/~mieum/relog/atom.xml => gemini://rawtext.club:1965/~mieum/music/atom.xml => gemini://gemini.thegonz.net:1965/glog/atom.xml => gemini://soviet.circumlunar.space:1965/wholesomedonut/gemlog/atom.xml => gemini://mozz.us:1965/journal/atom.xml => gemini://hannuhartikainen.fi:1965/twinlog/atom.xml => gemini://80h.dev:1965/glog/atom.xml => gemini://80h.dev:1965/~int/glog/atom.xml => gemini://exul.flounder.online/gemlog/atom.xml => gemini://latte.flounder.online/gemlog/atom.xml => gemini://pkill9.flounder.online/gemlog/atom.xml => gemini://sequel.space/bubble/XxLSRdjrZmctoE3CUmpnd2/atom.xml => gemini://sequel.space/bubble/VPxAbFt11WndPpYh1svnVS/atom.xml => gemini://sequel.space/bubble/V3pHHE5npaGdZ5cv88qMVq/atom.xml => gemini://inthetrees.flounder.online/gemlog/atom.xml => gemini://jordir.flounder.online/gemlog/atom.xml => gemini://ivanruvalcaba.cf/atom.xml => gemini://gemini.cyberbot.space:1965/gemlog/atom.xml => gemini://gemini.cyberbot.space:1965/smolzine/atom.xml => gemini://freeside.wntrmute.net:1965/log/index.rss => gemini://gemini.ucant.org:1965/gemlog/atom.xml => gemini://szczezuja.flounder.online/git/scripts/atom.xml => gemini://szczezuja.flounder.online/git/gmidiff/atom.xml => gemini://sjo.smol.pub/atom.xml => gemini://lyk.so/feed.atom => gemini://g.moi.lc/atom.xml => gemini://unixcat.coffee/gemlog/atom.xml => gemini://unixcat.coffee/recipes/atom.xml => gemini://armen138.flounder.online/gemlog/atom.xml => gemini://nicksphere.ch/atom.xml => gemini://frogs.flounder.online/gemlog/atom.xml => gemini://johnstephens.flounder.online/gemlog/atom.xml => gemini://stuserw.smol.pub/atom.xml => gemini://tilde.team/~konomo/atom.xml => gemini://blekksprut.net/日常鑑賞/atom.xml => gemini://bogart.flounder.online/gemlog/atom.xml => gemini://ruario.flounder.online/gemlog/atom.xml => gemini://lantashifiles.com/gemlog/entries/atom.xml => gemini://sotiris.papatheodorou.xyz/gemlog/atom.xml => gemini://vk3.wtf/gemlog/atom.xml => gemini://karabas.flounder.online/gemlog/atom.xml => gemini://sixohthree.com/atom.xml => gemini://bitdweller.flounder.online/gemlog/atom.xml => gemini://jdj.golf/gemlog/atom.xml => gemini://spxtr.net/atom.xml => gemini://yretek.com/atom.xml => gemini://gemini.ctrl-c.club/~pipe/atom.xml => gemini://low-key.me/gemlog/atom.xml => gemini://rasputin.selfip.net:1965/atom.xml => gemini://moddedbear.xyz:1965/logs/atom.xml => gemini://spool-five.com/gemlog/atom.xml => gemini://jfh.me:1965/atom.xml => gemini://going-flying.com:1965/thoughts/atom.xml => gemini://199.247.10.62/users/~julienxx/atom.xml => gemini://amadeus.flounder.online/gemlog/atom.xml => gemini://tentree.flounder.online/gemlog/atom.xml => gemini://hoseki.iensu.me/atom.xml => gemini://freeshell.de:1965/gemlog/atom.xml => gemini://koyu.space/blu256/atom.xml => gemini://dvd.flounder.online/feed.xml => gemini://sysnull.info/blog/atom.xml => gemini://www.groovestomp.com/gemlog/atom.xml => gemini://www.groovestomp.com/poetry/atom.xml => gemini://sequel.space/bubble/UcDBktgT6fVzeg7mhzSMMU/atom.xml => gemini://smol.pub/atom.xml => gemini://megymagy.flounder.online/gemlog/atom.xml => gemini://parsley.smol.pub/atom.xml => gemini://pietro.flounder.online/gemlog/atom.xml => gemini://sillylaird.flounder.online/gemlog/atom.xml => gemini://winter.flounder.online/gemlog/atom.xml => gemini://asciibene.flounder.online/gemlog/atom.xml => gemini://parsley.farm/atom.xml => gemini://cetacean.club:1965/journal/atom.xml => gemini://gluonspace.com/gemlog/atom.xml => gemini://kujiu.eu/blog/atom.xml => gemini://www.snonux.de/gemfeed/atom.xml => gemini://www.buetow.org/gemfeed/atom.xml => gemini://sequel.space/bubble/7nHibfwwMvZQvdt1LAA4P3/atom.xml => gemini://ruario.flounder.online/journal-atom.xml => gemini://text.adventuregameclub.com/atom.xml => gemini://foo.zone/gemfeed/atom.xml => gemini://www.foo.zone/gemfeed/atom.xml => gemini://gemini.circumlunar.space/news/atom.xml => gemini://gemini.circumlunar.space:1965/news/atom.xml => gemini://buetow.org:1965/gemfeed/atom.xml => gemini://nytpu.com:1965/flightlog/atom.xml => gemini://compudanzas.net:1965/atom.xml => gemini://alsd.eu:1965/it/feed.xml => gemini://calcuode.com:1965/gemlog/atom.xml => gemini://hedy.tilde.cafe:1965/feed.xml => gemini://salejandro.me:1965/gemlog/atom.xml => gemini://alsd.eu:1965/en/feed.xml => gemini://breadpunk.club:1965/~bakersdozen/gemlog/atom.xml => gemini://etam-software.eu:1965/blog/feed.xml => gemini://blog.schmidhuberj.de/atom.xml => gemini://tilde.team:1965/~konomo/atom.xml => gemini://tilde.pink:1965/~monerulo/atom.xml => gemini://dira.cc:1965/gemlog/atom.xml => gemini://corscada.uk/gemlog/atom.xml => gemini://bvnf.space/blog.rss => gemini://dns.eric.jetzt/gemlog/atom.xml => gemini://knijn.ga/posts/atom.xml => gemini://dctrud.randomroad.net/gemlog/atom.xml => gemini://cipay.ca/atom.xml => gemini://sanelkukic.smol.pub/atom.xml => gemini://costas.dev/posts/atom.xml => gemini://kayvr.com/gemlog/feed.xml => gemini://rawtext.club:1965/~verkaro/glog//atom.xml => gemini://sequel.space/bubble/8XvescodwwhNVjLVhV2HcP/atom.xml => gemini://dctrud.randomroad.net:1965/gemlog/atom.xml => gemini://cjc.im/feed.xml => gemini://mysidard.com/gemlog/public/atom.xml => gemini://colincogle.name/blog/atom.xml => gemini://evenstar.flounder.online/gemlog/atom.xml => gemini://saurabh.flounder.online/gemlog/atom.xml => gemini://tilde.club/~kerobaros/glog//atom.xml => gemini://hashtagueule.fr/atom.xml => gemini://armitage.flounder.online/gemlog/atom.xml => gemini://r2aze.observer/atom.xml => gemini://head.baselab.org/gemlog/atom.xml => gemini://0gitnick.flounder.online:1965/atom.xml => gemini://birabittoh.smol.pub:1965/atom.xml => gemini://blekksprut.net:1965/nikki/atom.xml => gemini://byzoni.org:1965/gemlog/atom.xml => gemini://capsule.usebox.net:1965/gemlog/atom.xml => gemini://causa-arcana.com:1965/blog/feed.xml => gemini://cdaniels.net:1965/feed_http.rss => gemini://cdaniels.net:1965/feed_http.atom => gemini://cdaniels.net:1965/feed_gmi.rss => gemini://cdaniels.net:1965/feed_gmi.atom => gemini://celehner.com:1965/feed.xml => gemini://cipay.ca:1965/atom.xml => gemini://dvd.flounder.online:1965/feed.xml => gemini://edwardtefft.com:1965/atom.xml => gemini://epi.benthic.zone:1965/atom.xml => gemini://g.dumke.me:1965/log/atom.xml => gemini://gluonspace.com:1965/gemlog/atom.xml => gemini://gmi.karl.berlin:1965/atom.xml => gemini://gsthnz.com:1965/posts/atom.xml => gemini://isoraqathedh.pollux.casa:1965/atom.xml => gemini://josias.dev:1965/gemlog/feed.xml => gemini://karmanyaah.malhotra.cc:1965/gemlog/atom.xml => gemini://makeworld.space:1965/gemlog/atom.xml => gemini://mysidard.com:1965/gemlog/public/atom.xml => gemini://nicksphere.com:1965/atom.xml => gemini://november.smol.pub:1965/atom.xml => gemini://nytpu.com:1965/gemlog/atom.xml => gemini://olivescout.flounder.online/gemlog/atom.xml => gemini://perplexing.space:1965/atom.xml => gemini://pulham.info:1965/atom.xml => gemini://rfmpie.smol.pub/atom.xml => gemini://ruario.flounder.online:1965/journal-atom.xml => gemini://rwv.io:1965/src/atom.xml => gemini://sixohthree.com:1965/atom.xml => gemini://testuser.flounder.online:1965/atom.xml => gemini://testuser.flounder.online:1965/myfeed.xml => gemini://thelambdalab.xyz:1965/atom.xml => gemini://unbinding.flounder.online/gemlog/atom.xml => gemini://vk3.wtf:1965/gemlog/atom.xml => gemini://yvngwytch840.flounder.online/gemlog/atom.xml => gemini://crash.smol.pub/atom.xml => gemini://longform.flounder.online/gemlog/atom.xml => gemini://binarycat.flounder.online/gemlog/atom.xml => gemini://malinfreeborn.com/gen/atom.xml => gemini://nafmusings.xyz/atom.xml => gemini://causa-arcana.com/blog/feed.xml => gemini://gemini.conman.org:1965/boston.atom => gemini://vectorprime.deszaras.xyz:1965/passages/atom.xml => gemini://kirill.zholnay.name/gemfeed/atom.xml => gemini://ttrpgs.org/all/atom.xml => gemini://seydaneen.nahtgards.de/leuchtturm/dwemerartefakte/gemini-circumlunar-space/news/atom.xml => gemini://gemini.locrian.zone/gemini/news/atom.xml => gemini://gemini.locrian.zone:1965/gemini/news/atom.xml => gemini://gemini.circumlunar.space/users/rbasile/blog/feed.atom => gemini://cap.swan.quest/p/atom.xml => gemini://rawtext.club:1965/~mieum/dallok/atom.xml # CAPCOM: => gemini://voidcruiser.nl => gemini://station.martinrue.com/martin/4c5f8b1291a049319033ad1c33aa373d => gemini://g.xn--nck.club/feed.xml => gemini://mozz.us/journal/atom.xml => gemini://gemlog.blue/users/futagoza/1619481193.gmi => gemini://monmac.flounder.online => gemini://gem.acdw.net/do/rss => gemini://otrn.org/atom.xml => gemini://mrnd.xyz/log/atom.xml => gemini://freeshell.de/gemlog/atom.xml => gemini://nuclear.discrust.pl => gemini://posixcafe.org/blogs/feed.atom => gemini://republic.circumlunar.space/users/dbane/gemlog/atom.xml => gemini://overeducated-redneck.net/glog/atom.xml => gemini://nalie.art => gemini://midnight.pub/feed.xml => gemini://decelerationist.net/feed.xml => gemini://gemini.bvnf.space/blog.rss => gemini://t-900.flounder.online/gemlog/atom.xml => gemini://calcuode.com/gemlog/atom.xml => gemini://calmwaters.xyz => gemini://six10.pw => gemini://sotiris.papatheodorou.xyz/gemlog/atom.xml => gemini://gemini.susa.net/atom.xml => gemini://gem.informethique.org/recettes/atom.xml => gemini://om.gay/redacted/gmi.atom => gemini://foo.zone/gemfeed/atom.xml => gemini://thesudorm.com/atom.xml => gemini://gem.rmgr.dev/blog/atom.xml => gemini://luke.flounder.online/gemlog/atom.xml => gemini://gemini.ctrl-c.club/~stack/gemlog/ => gemini://gemini.kevinronceray.com/atom.xml => gemini://hannuhartikainen.fi/twinlog/atom.xml => gemini://fortunebot.crabdance.com/feed/atom.xml => gemini://rawtext.club/~tolstoevsky/glog/atom.xml => gemini://tilde.team/~remyabel/atom.xml => gemini://ybad.name/log/feed.atom => gemini://soviet.circumlunar.space/~hektor/atom.xml => gemini://xj-ix.luxe:1969/feed.atom => gemini://pietro.flounder.online/gemlog => gemini://soviet.circumlunar.space/wholesomedonut/gemlog/atom.xml => gemini://jlk.flounder.online/gemlog/ => gemini://l-3.space/atom.xml => gemini://cdaniels.net/feed_gmi.atom => gemini://republic.circumlunar.space/users/crdpa/blog/atom.xml => gemini://bcn08012.ddns.net/atom.xml => gemini://blog.hispagatos.org => gemini://gemini.marmaladefoo.com/cgi-bin/atom-feed.cgi?lukee => gemini://simbly.me/posts/atom.xml => gemini://idf.looting.uk/capslog/atom.xml => gemini://oberdada.pollux.casa/gemlog.gmi => gemini://tilde.club/~lewiscowper/gemlog/atom.xml => gemini://gemlog.blue/users/cariboudatascience/1613780342.gmi => gemini://avalos.me/gemlog/atom.xml => gemini://inconsistentuniverse.space/atom.xml => gemini://carcosa.net/journal/atom.xml => gemini://tilde.team/~steve/atom.xml => gemini://www.demorrow.net/atom.xml => gemini://republic.circumlunar.space/crdpa/blog/atom.xml => gemini://skyjake.fi/gemlog/atom.xml => gemini://wirthslaw.flounder.online/gemlog/atom.xml => gemini://gemini.maxxk.me/atom.xml => gemini://sdf.org/atom.xml => gemini://rawtext.club/~ploum/atom.xml => gemini://benj.smol.pub/atom.xml => gemini://l-ipa.net/atom.xml => gemini://gemini.go350.com/atom.xml => gemini://subphase.xyz/users/flow/gemlog/atom.xml => gemini://gemlog.blue/users/cariboudatascience/ => gemini://mikelynch.org/index.gmi => gemini://gemini.strahinja.org/blog/rss.xml => gemini://quasivoid.net/gemlog/atom.xml => gemini://alsd.eu/en/feed.xml => gemini://hugeping.tk/atom.xml => gemini://ondollo.com/~/zert:7C:8E:F5/atom.gem => gemini://g.dumke.me/log/atom.xml => gemini://tilde.team/~easeout/glog/atom.xml => gemini://gemini.bbbhltz.space/gemlog/atom.xml => gemini://tilde.team/~ivanruvalcaba/glog/atom.xml => gemini://alexschroeder.ch:1965/do/blog/atom => gemini://starbreaker.org/atom.xml => gemini://blog.snowfrost.garden/atom.xml => gemini://blog.schmidhuberj.de/atom.xml => gemini://cadadr.space/blag.gmi => gemini://gemini.ctrl-c.club/~lettuce => gemini://bbs.archaicbinary.net/atom.xml => gemini://snowcode.ovh/rss.xml => gemini://ainent.xyz/gemlog/index.gmi => gemini://phreedom.club/~tolstoevsky/glog/atom.xml => gemini://gems.geminet.org/avr/atom.xml => gemini://0x80.org/gemlog/atom.xml => gemini://gemini.rawles.net/blog/atom.xml => gemini://josias.dev/gemlog/feed.xml => gemini://sbg.one/SandboxGeneral/atom.xml => gemini://gemini.freeradical.zone/log/ => gemini://gem.iwritethe.codes/posts/atom.xml => gemini://dira.cc/gemlog/atom.xml => gemini://figbert.com/log/atom.xml => gemini://lantashifiles.com/gemlog/entries/atom.xml => gemini://sl1200.dystopic.world/feed.gmi
Gemini/gemini-scripts-README.txt
# 2021-02-16 All these scripts are very specific to Techright's structure. Though ideas and components might be reusable elsewhere with modification. Each involves /lots/ of scraping and are therefore inherently brittle. They are run twice a day via a shell script in cron: $ crontab -l | grep -E -v '^#' 01 */3 * * * /home/gemini/bin/gemini-cron-updater.sh That script contains the work flow for all the steps needed to keep the Gemini site up to date. So reading it would be the place to start. # gemini-fetch-urls-from-rss.pl This gets an RSS feed and extracts the date. RPiOIS has a quirk in regards to /dev/stdout ownership, so that part needs repair. However, the -w option works. This script is standalone but can pipe data to another script or pass it via a file. Probably this ought to be redone to use wget instead. Example: ./gemini-fetch-urls-from-rss.pl http://techrights.org/feed/ Example: ./gemini-fetch-urls-from-rss.pl -d 2021-02-12 http://techrights.org/feed/ Example: ./gemini-fetch-urls-from-rss.pl -w tr.urls.txt -d 2021-02-12 \ http://techrights.org/feed/ # gemini-fetch-web-page.pl Reads in a URL and fetches them using wget. wget is more flexible than CPAN's LWP. This script is not standalone, it calls the next script, gemini-parse-html-to-gemini.pl using a temporary file to pass the data. It has the option to convert some of the links to Gemini links. Example: cat ~/bin/feeds.tr.2021.txt \ | xargs ~/bin/gemini-fetch-web-page.pl -g -p -v -b /home/gemini/gemini/ # gemini-parse-html-to-gemini.pl Reads a file containing XHTML and converts it to Gemini mark up, with the expectation that it conforms to the structures used at Techrights as of 2021. This is a standalone script. Example: ./gemini-parse-html-to-gemini.pl -w /dev/stdout -v sample.html # gemini-inventory.pl Traverses the file system and extracts titles from the Gemini articles. The data is used to construct daily and monthly summaries. Example: ./gemini-inventory.pl /home/gemini/gemini/2021/ ## perl modules XPath is used for most of the processing. Cwd English File::Basename File::Glob File::Path File::Temp Getopt::Std HTML::TreeBuilder::XPath HTTP::Response::Encoding LWP::UserAgent Path::Iterator::Rule POSIX Time::ParseDate Time::Piece URI utf8 XML::Feed # gemini-bulletin-irc-update.sh Creates Gemini indexes from the text bulletins. The bulletins will be read from the ~/gemini/tr_text_versioni/ directory. # gemini-cron-updater.sh A batch job to update starting by reading the RSS feed and finishing with updating the indexes and bulletins. # gemini-main-index-template.sh A template for making the 'home page' for the Gemini site
Gemini/wget-tr-pages.sh
#!/bin/sh
# 2021-02-22 download TR back issue articles a year at a
# time and then convert them from HTML to Gemini
year=$1
echo $year | sed -r '/^[0-9]{4}$/q; s/.*/0/'
if [ $year -eq 0 ]; then
echo "Year missing"
exit 1
fi
stop=$1
echo $stop | sed -r '/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/q; s/.*/0/'
target=/home/gemini
cache=${target}/X/
log=/tmp/wget.tr.log
# ---
PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin
test -f ${log} && rm -f ${log}
# the wget loops can take from 5 to 13 hours depending on
# the number of posts that year
for m in $(seq --format '%02.0f' 11 12)
do
for d in $(seq --format '%02.0f' 1 31);
do
date -d "$year-$m-$d" > /dev/null || break
wget \
--progress=dot \
--limit-rate=90k \
--wait=4 \
--random-wait \
--tries=22 \
--retry-on-http-error=500,503 \
--no-host-directories \
--directory-prefix=${cache} \
--recursive \
--level=3 \
--no-parent \
--accept '*.html' \
http://techrights.org/${year}/$m/$d/ \
| tee -a ${log}
done
done
# find all the cached web pages and send them to the parser
# for conversion to Gemini format; use parallel to run several
# instances of the parser concurrently
time \
find ${cache} \
-type f \
-name '*.html' \
-not -path '*/page/*' \
-not -path '*/feed/*' \
-print0 | \
parallel \
--null \
--jobs 6 gemini-fetch-web-page.pl \
-p -b ${target} file://{} :::
# see also:
# https://www.gnu.org/software/parallel/parallel_cheat.pdf
exit 0
Gemini/purge_logs.sh
# To protect people's privacy run near end of each month; don't delete everything for a number of reasons, including open files and spacing anomalies echo 'Log purge of the following files:' ls -la ../logs/gemini-log-2023-05* rm -f ../logs/gemini-log-2023-05-3* rm -f ../logs/gemini-log-2023-05-0* rm -f ../logs/gemini-log-2023-05-1* rm -f ../logs/gemini-log-2023-05-2* echo "C'est la vie"
Gemini/tr-archive-stats.sh
#!/bin/sh
# archive stats page
# will need to be extended at some point to partition many pages
g=/home/gemini/techrights.org
d=$(date +"%Y-%m-%d")
set -e
cp ${g}/stats/index.gmi ${g}/stats/stats$(date -d ${d} +"-%Y-%m-%d").gmi
sed -i "1s|^|$(date +"# %d %m %Y Archive\n")|" \
${g}/stats/stats$(date -d ${d} +"-%Y-%m-%d").gmi
exit 0
Gemini/.directory-listing-ok
Gemini/gemini-fetch-web-page.pl
#!/usr/bin/perl -T
# 2021-02-17 Fetches HTML from URLs and
# calls another script to convert to Gemini
# Alpha Version
# Reads the URLs from the command line
use utf8;
use Getopt::Std;
use Cwd qw(getcwd abs_path);
use URI;
use File::Path qw(make_path);
use File::Basename;
use File::Temp qw(tempfile);
use HTML::TreeBuilder::XPath;
use English;
use warnings;
use strict;
$OUTPUT_AUTOFLUSH = 1;
# untaint the $PATH
$ENV{'PATH'} = '/home/gemini/bin:/usr/local/bin:/usr/bin:/bin';
# path to the next script, it will convert XHTML to Gemini and save it
# my $parser = './gemini-parse-html-to-gemini.pl';
my $parser = 'gemini-parse-html-to-gemini.pl';
# command line option
our %opt;
getopts('b:ghmpv', \%opt);
&usage if ($opt{'h'});
# set the base working directory for resulting files,
# it will be created later on if necessary
my $dest_path;
if ($opt{'b'}) {
$dest_path = $opt{'b'};
}
# sanity checking of destination path
unless($dest_path) {
warn("Use -b to set a destination path.\n\n");
&usage;
}
if ($dest_path =~ m{^/?.*?(?=\.\.)}) {
die("Use an absolute path.\n\n");
} elsif ($dest_path =~ m{^(/[\w\-\=\%\.\/]+)}) {
$dest_path = $1;
$dest_path =~ s|([^/])$|$1/|;
} else {
die("Wonky path: '$dest_path'\n\n");
}
$dest_path = abs_path($dest_path);
if( ! $dest_path){
die("Bad destination path.\n\n");
} elsif (-d $dest_path && ! -w $dest_path) {
warn("Path: '$dest_path'\n");
die("Destination path is not writable: '$dest_path' \n\n");
} elsif (! -d $dest_path && ! $opt{'p'}) {
warn("Path: '$dest_path'\n");
die("Destination path doesn't exist. Use -p also.\n\n");
}
# process the URLs passed as command line parameters
my @URL = ();
foreach my $url (@ARGV) {
my $uri = URI->new($url);
if (defined($uri->scheme)) {
if ($uri->scheme eq 'http'
or $uri->scheme eq 'https'
or $uri->scheme eq 'file') {
push(@URL, $uri);
}
}
}
&usage if ($#URL < 0);
# process each URL, one at a time
foreach my $url (@URL) {
if ($opt{'v'}) { print $url->canonical,qq(\n); }
# get the document path from the URL
my $path = $dest_path.'/'.$url->path;
( $path ) = ($path =~ m/^([\w\-\.\/\%]+)$/); # untaint
while ($path =~ s|//|/|g) { 1 }
my $name = '';
($name, $path) = fileparse($path);
if ($name) {
$name =~ s/\.html$//;
$name .= '.gmi';
($name) = ($name =~ m/^([\w\-\.\/\%]+)$/); # untaint
} else {
$name = 'index.gmi';
}
if($opt{'v'}) {
print qq(Path = $path\n);
print qq(Name = $name\n);
}
# use a temp file to get the XHTML over to the next script
my $tmp = File::Temp->new( TEMPLATE => 'temp.XXXXX',
DIR => '/tmp',
SUFFIX => '.fetch.gemini.tmp',
UNLINK => 1 );
my $tmpfile = $tmp->filename;
-f $tmpfile && unlink($tmpfile); # clear the way for wget
my $success = 0;
if ($url->scheme eq 'http' or $url->scheme eq 'https') {
# fetch URL and hold content in a variable
$success = &fetch($url, $tmpfile);
# if the page content and destination path exist, continue
if ( $success && $path) {
# make the target directory if called for
unless(-d $path or ! $opt{'p'}) {
if ($path =~ m{^(/[\w\-\=\%\.\/]+)}) {
$path = $1;
}
make_path($path,{mode=>0755})
or die("Could not create path '$path' : $!\n");
}
# parse temp file and write output to "$path$name"
my @args = ();
push(@args, '-g') if ($opt{'g'}); # tell it to convert links
my @cmd = ($parser, @args, '-w', "$path$name", $tmpfile);
system(@cmd) == 0
or die("system '@cmd' failed: $?\n");
# fetching videos is not done by default
if($opt{'m'}) {
print qq(Fetching videos\n) if ($opt{'v'});
$success = &extract_webm($url, $tmpfile, $path);
}
# close temp file, if UNLINK is set then it is deleted now
close($tmp);
} else {
exit(1);
}
} elsif ($url->scheme eq 'file') {
print qq(FILE:///\n) if ($opt{'v'});
$success = &fetch_file($url, $tmpfile);
$path = $url->path;
# hard-coded /yyyy/mm/dd/ pattern ...
if($path =~ m{^.*/(\d{4}/\d{2}/\d{2}/.*)/[^/]+\.html}) {
$path = $dest_path.'/'.$1;
} else {
exit(0);
}
print qq(DEST_PATH=$path\n) if ($opt{'v'});
# if the page content and destination path exist, continue
if ( $success && $path) {
# make the target directory if called for
unless(-d $path or ! $opt{'p'}) {
if ($path =~ m{^(/[\w\-\=\%\.\/]+)}) {
$path = $1;
}
make_path($path,{mode=>0755})
or die("Could not create path '$path' : $!\n");
}
# parse temp file and write output to "$path$name"
my @args = ();
push(@args, '-g') if ($opt{'g'}); # tell it to convert links
my @cmd = ($parser, @args, '-w', "$path/$name", $tmpfile);
print qq(CMD=@cmd\n) if($opt{'v'});
system(@cmd) == 0
or die("system '@cmd' failed: $?\n");
# fetching videos is not done
# close temp file, if UNLINK is set then it is deleted now
close($tmp);
} else {
exit(1);
}
}
}
exit(0);
sub usage {
print qq(Fetch web pages (from TechRights) for conversion to Gemini\n);
$0 =~ s/^.*\///;
print qq($0: [-hbmpv] URL [URL...]\n);
print qq( -h print this help text\n);
print qq( -b base working directory, default is current\n);
print qq( -m also fetch and store videos, default is not to\n);
print qq( -p create destination directories\n);
print qq( -v increase message verbosity\n);
exit(1);
}
sub fetch_file {
my ( $uri, $filename ) = ( @_ );
my $path = $uri->path; # works only with absolute paths
print qq(SRC FILE=$path\n) if ($opt{'v'});
unless(-f $path && -r $path){
warn("File '$path' does not exist or is unreadable!\n");
return(0);
}
my $result = system('cp', $path, $filename);
if($result) {
die("File copy failed,\n");
}
return(1);
}
sub fetch {
my ( $uri, $filename ) = ( @_ );
# wget is more flexible than LWP,
# especially with large files like the videos
my @args=(
'--quiet',
'--user-agent=TechRights-Gemini-Bot/0.1',
'--no-directories',
'--no-clobber',
'--retry-on-http-error=500,503',
'--tries=25',
'--waitretry=8',
"--output-document=$filename"
);
my $url = $uri->canonical;
print qq(URI Canonical = ), $url,qq(\n) if ($opt{'v'});
$url =~ s/\#.*$//;
$url =~ s/\?.*$//;
unless (( $url ) = ( $url =~ m/^([\w\.\-\%\/\:]+)$/ )) {
print qq(WrongURL = $uri->canonical\n);
return(0);
}
print qq(Fetching $url\n) if ($opt{'v'});
my @cmd = ('wget', @args, $url);
my $result = system(@cmd);
if($result == 256) {
warn("File '$filename' already exists!\n");
} elsif ($result) {
die("system '@cmd' failed: $?\n");
}
return(1);
}
sub extract_webm {
my ($base, $tmpfile, $path) = (@_);
my $xhtml = HTML::TreeBuilder::XPath->new;
$xhtml->implicit_tags(1);
$xhtml->no_space_compacting(0);
$xhtml->parse_file($tmpfile);
my %videos = ();
my %ok_hosts = (
'www.techrights.org' => 1,
'techrights.org' => 1,
);
for my $a ($xhtml->findnodes('//a[contains(@href,".webm")]')) {
my $url = URI->new($a->attr('href'));
$url = $url->abs($base); # convert relative links
if (defined($url->host) && $ok_hosts{$url->host}) {
$videos{$url->canonical} = $url;
}
}
foreach my $vurl (sort values %videos) {
# get the document path from the URL
my $path = $dest_path.'/'.$vurl->path;
( $path ) = ($path =~ m/^([\w\-\.\/\%]+)$/); # untaint
while ($path =~ s|//|/|g) { 1 }
my $name = '';
($name, $path) = fileparse($path);
if ($name =~ m/\.webm/) {
($name) = ($name =~ m/^([\w\-\.\/\%]+)$/); # untaint
} else {
return(0);
}
if (! -d $path and $opt{'p'}) {
if ($path =~ m{^(/[\w\-\=\%\.\/]+)}) {
$path = $1;
}
make_path($path,{mode=>0755})
or die("Could not create path '$path' : $!\n");
} elsif (! -d $path) {
die("Path '$path' is missing!\n");
}
&fetch($vurl, "$path$name");
}
}
Gemini/gemini-bulletin-irc-update.sh
#!/bin/bash
# 2021-02-14 CreateGemini indexes for various IRC logs
# for History see Git archive
#
p=/home/gemini/techrights.org; # path to working directory
limit=5; # cycle back this many months
# #####
# main body of program
PATH=/usr/local/bin:/usr/bin:/bin
set -e
cd $p
now=$(date +"%Y-%m-%d")
# construct Gemini indexes for Bulletins
m=0
while test $m -lt $limit
do
dd=$(date +"%Y-%m" -d "$now -$m month")
bb=$(date +"%Y/%m" -d "$now -$m month")
test -e ./bulletins/$bb/ || mkdir -p ./bulletins/$bb/
cp ~gemini/bulletin-menu.txt ./bulletins/$bb/index.gmi
find ./tr_text_version -name "techrights-$dd*" -print | sort \
| sed -re 's|^\.([^-]+)-(.*).txt|=> \1-\2.txt Techrights \2|;' \
>> ./bulletins/$bb/index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> ./bulletins/$bb/index.gmi
cat ~gemini/logo.txt >> ./bulletins/$bb/index.gmi
m=$((m+1))
done
# construct Gemini indexes for IRC logs
m=0
while test $m -lt $limit
do
mm=$(date +"%m" -d "$now -$m month")
yy=$(date +"%y" -d "$now -$m month")
yyyy=$(date +"%Y" -d "$now -$m month")
test -e ./irc/$yyyy/$mm/ || mkdir -p ./irc/$yyyy/$mm/
cp ~gemini/irc-menu.txt ./irc/$yyyy/$mm/index.gmi
echo '## GemText Version' >> ./irc/$yyyy/$mm/index.gmi
find ./irc-gmi -name "*irc*techrights*$mm$yy.gmi" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Gemini/GemText IRC Logs for Techrights $yyyy-$mm-\2|" \
>> ./irc/$yyyy/$mm/index.gmi
echo '## Plain Text Version' >> ./irc/$yyyy/$mm/index.gmi
find ./tr_text_version -name "*irc*techrights*$mm$yy.txt" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Plain Text IRC Logs for Techrights $yyyy-$mm-\2|" \
>> ./irc/$yyyy/$mm/index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> ./irc/$yyyy/$mm/index.gmi
cat ~gemini/logo.txt >> ./irc/$yyyy/$mm/index.gmi
m=$((m+1))
done
# construct Gemini indexes for TR Social IRC logs
m=0
while test $m -lt $limit
do
mm=$(date +"%m" -d "$now -$m month")
yy=$(date +"%y" -d "$now -$m month")
yyyy=$(date +"%Y" -d "$now -$m month")
test -e ./social/$yyyy/$mm/ || mkdir -p ./social/$yyyy/$mm/
cp ~gemini/irc-menu.txt ./social/$yyyy/$mm/index.gmi
echo '## GemText Version' >> ./social/$yyyy/$mm/index.gmi
find ./irc-gmi -name "*social*$mm$yy.gmi" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Gemini/GemText IRC Logs for Techrights Social $yyyy-$mm-\2|" \
>> ./social/$yyyy/$mm/index.gmi
echo '## Plain Text Version' >> ./social/$yyyy/$mm/index.gmi
find ./tr_text_version -name "*social*$mm$yy.txt" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Plain Text IRC Logs for Techrights Social $yyyy-$mm-\2|" \
>> ./social/$yyyy/$mm/index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> ./social/$yyyy/$mm/index.gmi
cat ~gemini/logo.txt >> ./social/$yyyy/$mm/index.gmi
m=$((m+1))
done
# construct Gemini indexes for TR TechBytes IRC logs
m=0
while test $m -lt $limit
do
mm=$(date +"%m" -d "$now -$m month")
yy=$(date +"%y" -d "$now -$m month")
yyyy=$(date +"%Y" -d "$now -$m month")
test -e ./techbytes/$yyyy/$mm/ || mkdir -p ./techbytes/$yyyy/$mm/
cp ~gemini/irc-menu.txt ./techbytes/$yyyy/$mm/index.gmi
echo '## GemText Version' >> ./techbytes/$yyyy/$mm/index.gmi
find ./irc-gmi -name "*techbytes*$mm$yy.gmi" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Gemini/GemText IRC Logs for Techrights TechBytes $yyyy-$mm-\2|" \
>> ./techbytes/$yyyy/$mm/index.gmi
echo '## Plain Text Version' >> ./techbytes/$yyyy/$mm/index.gmi
find ./tr_text_version -name "*techbytes*$mm$yy.txt" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Plain Text IRC Logs for Techrights TechBytes $yyyy-$mm-\2|" \
>> ./techbytes/$yyyy/$mm/index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> ./techbytes/$yyyy/$mm/index.gmi
cat ~gemini/logo.txt >> ./techbytes/$yyyy/$mm/index.gmi
m=$((m+1))
done
# construct Gemini indexes for TR BoycottNovell IRC logs
m=0
while test $m -lt $limit
do
mm=$(date +"%m" -d "$now -$m month")
yy=$(date +"%y" -d "$now -$m month")
yyyy=$(date +"%Y" -d "$now -$m month")
test -e ./boycottnovell/$yyyy/$mm/ || mkdir -p ./boycottnovell/$yyyy/$mm/
cp ~gemini/irc-menu.txt ./boycottnovell/$yyyy/$mm/index.gmi
# find ./tr_text_version -name "irc-log-*$mm$yy.txt" -print \
echo '## GemText Version' >> ./boycottnovell/$yyyy/$mm/index.gmi
find ./irc-gmi -regextype posix-basic \
-regex ".*/irc-log-[0-9]\{2\}$mm$yy.gmi\$" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Gemini/GemText IRC Logs for Techrights BoycottNovell $yyyy-$mm-\2|" \
>> ./boycottnovell/$yyyy/$mm/index.gmi
echo '## Plain Text Version' >> ./boycottnovell/$yyyy/$mm/index.gmi
find ./tr_text_version -regextype posix-basic \
-regex ".*/irc-log-[0-9]\{2\}$mm$yy.txt\$" -print \
| sort \
| sed -r -e "s|^\.(.*)-(..)(.*)$|=> \1-\2\3 Plain Text IRC Logs for Techrights BoycottNovell $yyyy-$mm-\2|" \
>> ./boycottnovell/$yyyy/$mm/index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" \
>> ./boycottnovell/$yyyy/$mm/index.gmi
cat ~gemini/logo.txt \
>> ./boycottnovell/$yyyy/$mm/index.gmi
m=$((m+1))
done
cd $p/irc/
cp ~gemini/irc-menu.txt index.gmi
echo '## Logs for #techrights at irc.techrights.org' >> index.gmi
echo >> index.gmi
find . -mindepth 2 -maxdepth 2 -type d \
| sort -r \
| cut -b 3-100 \
| while read -r line; do \
echo -n "=> $line/index.gmi " & sed "s/\/index.gmi//" \
<<< "$line"
done >> index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> index.gmi
cat ~gemini/logo.txt >> index.gmi
cd $p/social/
cp ~gemini/irc-menu.txt index.gmi
echo '## Logs for #boycottnovell-social at irc.techrights.org' >> index.gmi
echo >> index.gmi
find . -mindepth 2 -maxdepth 2 -type d \
| sort -r \
| cut -b 3-100 \
| while read -r line; do \
echo -n "=> $line/index.gmi " & sed "s/\/index.gmi//" \
<<< "$line"
done >> index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> index.gmi
cat ~gemini/logo.txt >> index.gmi
cd $p/techbytes/
cp ~gemini/irc-menu.txt index.gmi
echo '## Logs for #techbytes at irc.techrights.org' >> index.gmi
echo >> index.gmi
find . -mindepth 2 -maxdepth 2 -type d \
| sort -r \
| cut -b 3-100 \
| while read -r line; do \
echo -n "=> $line/index.gmi " & sed "s/\/index.gmi//" \
<<< "$line"
done >> index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> index.gmi
cat ~gemini/logo.txt >> index.gmi
cd $p/boycottnovell/
cp ~gemini/irc-menu.txt index.gmi
echo '## Logs for #boycottnovell at irc.techrights.org' >> index.gmi
echo >> index.gmi
find . -mindepth 2 -maxdepth 2 -type d \
| sort -r \
| cut -b 3-100 \
| while read -r line; do \
echo -n "=> $line/index.gmi " & sed "s/\/index.gmi//" \
<<< "$line"
done >> index.gmi
date -u +"%nUpdated %F %H:%M UTC%n" >> index.gmi
cat ~gemini/logo.txt >> index.gmi
exit 0
Gemini/gemini-make-feed-wrapper.sh
#!/bin/sh
PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin
# percent signs are processed in crontabs so this wrapper is needed
gemini-make-feed.pl \
-r /home/gemini/gemini \
-o /home/gemini/gemini/feed.xml \
/home/gemini/gemini/$(date +"%Y/%m")/ \
> /home/gemini/logs/make-feed.log
exit 0
Gemini/irc.gmi
Welcome to Techrights IRC section => / Back to index # IRC Introduction Techrights has its own IRC network. => about/ To access our network see details in this page # IRC Logs in Techrights => chat/index.gmi ▢ - Latest chat logs for #techrights (updated every 5-10 minutes) ### Techrights Archive => irc/index.gmi ㏒ - Logs for #techrights ### Social Archive => social/index.gmi ㏒ - Logs for #boycottnovell-social ### Techbytes Archive => techbytes/index.gmi ㏒ - Logs for #techbytes ### BoycottNovell Archive => boycottnovell/index.gmi ㏒ - Logs for #boycottnovell => /logo/logo.png Site logo (if your Gemini client supports that) ### ➮ Sharing is caring. Content is available under CC-BY-SA. ⟲
Gemini/agate-tcpdump-logger.sh
#!/bin/sh
# 2022-01-22
PATH=/usr/sbin:/usr/bin:/sbin:/bin
today=$(date +"%F")
echo '\n----------------------------------------------------------------' \
>> /home/gemini/logs/gemini-log-${today}.log
date +"Restarting logging at %F %T%n" \
>> /home/gemini/logs/gemini-log-${today}.log
# use wlan0 below if the server uses Wi-Fi
stdbuf -oL tcpdump -ttttqpli eth0 'inbound and port 1965
and not src net 192.168.4.0/24
and tcp[tcpflags] & tcp-syn != 0' \
| perl -a -n -e '
$|=1; # autoflush buffer
$d=$F[0]; $t=$F[1]; $h=$F[3];
$t=~s/\.[0-9]+$//; $h=~s/\.\d+$//;
# print STDERR qq($d $t $h\n);
print qq($d $t $h\n);
' \
>> /home/gemini/logs/gemini-log-${today}.log
exit 0
Gemini/RPi-Manchester/show-new-visitors-count.sh
#!/bin/sh
# 2021-02-27
PATH=/usr/bin:/bin
set -e
awk '$3 {a[$3]++} END{ for (b in a) {print a[b],b}}' OFS="\t" \
/home/gemini/logs/gemini-log-$(date +"%F").log \
| sort -k1,1n -k2,2
exit 0
Gemini/RPi-Manchester/tcpdump-logger.sh
#!/bin/sh
PATH=/usr/sbin:/usr/bin:/sbin:/bin
now=$(date +"%F")
echo '----------------------------------------------------------------------------' >> /home/gemini/logs/gemini-log-${now}.log
echo -n 'Restarting logging at ' >> /home/gemini/logs/gemini-log-${now}.log
date >> /home/gemini/logs/gemini-log-${now}.log
echo '' >> /home/gemini/logs/gemini-log-${now}.log
tcpdump \
-q \
-p \
-l \
-tttt \
-i wlan0 \
'not src net 192.168.1.0/24 and dst net 192.168.1.0/24
and tcp[tcpflags] & (tcp-syn) != 0 and port 1965' \
| awk '{
sub(/\.[0-9]+$/,"",$2); \
sub(/\.[0-9]+$/,"",$4); \
print $1, $2, $4; \
fflush();\
}' >> /home/gemini/logs/gemini-log-${now}.log
Gemini/RPi-Manchester/gemini-tcpdump-logger.service
[Unit] Description=Use tcpdump to log contavts to the Gemini server After=network.target [Service] User=root Type=exec ExecStart=/usr/local/sbin/tcpdump-logger.sh ExecReload=/bin/kill -HUP $MAINPID KillMode=control-group Restart=on-failure RestartPreventExitStatus=255 RestartSec=5s [Install] WantedBy=multi-user.target
Gemini/RPi-Manchester/.directory-listing-ok
Gemini/RPi-Manchester/show-new-visitors.sh
#!/bin/sh
# 2021-02-13
# 2021-02-27 updated
# tail -n 100 -f log.txt \
# | grep -v host81 \
# | grep -v roibng \
# | grep -v '\-\-'
# host81-154-168-60.range81-154.btce:44808 <= 0b 739b 462b 924B
# [2021-02-26 04:29:59] v2202102141844143923.supersrv.de:53510 <= 0b 0b 266b 1.30KB
PATH=/usr/bin:/bin
# tail -f /home/gemini/logs/gemini-$(date +"%F").log \
# exit 0
tail -n 500 -f /home/gemini/logs/gemini-log-$(date +"%F").log \
| awk '($5!="0b" || $6!="0b" || $7!="0b" || $8!="0b") \
&& $3!~/roibng/ \
&& $3!~/btce/ \
&& $3~/[a-z0-9]/ { \
sub(/:.*$/, "", $3); \
print $1 " " $2, $3; \
}'
Gemini/tr-gemini-latest-videos.sh
#!/bin/sh # update latest video gallery page PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin d=/var/www/techrights.org/htdocs g=/home/gemini/techrights.org cat $d/videos/index.html | tr-gemini-latest-videos.pl > $g/videos/index.gmi exit 0
Gemini/archive-stats.sh
#!/bin/sh # archive stats page # will need to be extended at some point to partition many pages cp /home/gemini/gemini/stats/index.gmi /home/gemini/gemini/stats/stats$(date +"-%Y-%m-%d").gmi && sed -i "1s|^|$(date +"# %d %m %Y Archive\n")|" /home/gemini/gemini/stats/stats$(date +"-%Y-%m-%d").gmi
Gemini/gemini-ping-roy.sh
#!/bin/sh # 2020-12-25 PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin if test "x$1" = "xbright"; then timeout 9 run-blinkt.py flashed_bar 250 255 255 0.0001 else timeout 9 run-blinkt.py flashed_bar 250 128 128 0.0001 fi exit 0
Gemini/gemini-latest-videos.sh
#!/bin/sh # 2021-09-17 # update latest video gallery page PATH=/home/gemini/bin:/usr/local/bin:/usr/bin:/bin sudo /usr/local/sbin/tc-shaper-v2021-Nov.sh wget -qO- --tries=21 --waitretry=10 http://techrights.org/videos/ \ | gemini-latest-videos.pl > ~/gemini/latest-videos/index.gmi sudo /usr/local/sbin/tc-shaper-v2.sh exit 0
Gemini/tr-gemini-git-sloc.sh
#!/bin/sh
# 2021-10-09
PATH=/usr/local/bin:/usr/bin:/bin
d=/home/gemini/techrights.org/git/sloc.gmi
s=/home/gemini/techrights.org/git/tr-git/
echo "# SLOC stats $(date +'%F')\n" > ${d}
echo "Source Lines Of Code (SLOC) it not a useful metric and provideded \
only for amusement.\n" >> ${d}
sloccount ${s} 2>/dev/null \
| sed -n -r -e '
/^SLOC/,/^$/ {s/^SLOC./# SLOC /; s/SLOC-/ &/;p};
/^Totals/,/^$/ {s/^Totals/# &/;p};
/^Total / {p;q}' \
>> ${d}
cat << EOT >> ${d}
## Licence
GNU Affero General Public License (AGPLv3) unless stated otherwise, e.g. Public Domain
=> / back to Techrights (Main Index)
EOT
Gemini/gemini-latest-videos.pl
#!/usr/bin/perl
# 2021-09-17
# update latest video gallery page
use HTML::TreeBuilder::XPath;
use File::Basename;
use JSON;
print "# Latest Techrights Videos\n\n";
print "Most recent shown first (the list below is limited to past 7 days)\n\n";
my %videos = &xhtml_video_links('-');
my %metadata = &fetch_metadata(%videos);
foreach my $key (sort {$a<=>$b} keys %metadata) {
print "=> ";
print $metadata{$key}{'link'}," ";
print "↺ ";
printf ("%2d ",$key);
if (exists($metadata{$key}{'title'})) {
print $metadata{$key}{'title'};
if(exists($metadata{$key}{'date'})){
print " (",$metadata{$key}{'date'},")";
}
} else {
print $metadata{$key}{'file'};
}
print "\n";
}
print "\n=> /videos/ Show videos archive\n";
print "=> / Back to homepage\n";
exit(0);
sub xhtml_video_links{
my ($file)= (@_);
my $is_stdin = 0;
my $input;
if ($file eq '-') {
$input = *STDIN;
$is_stdin++;
} else {
# force input to be read in as UTF-8
open ($input, "<:utf8", $file)
or die("Could not open file '$file' : error: $!\n");
}
my $xhtml = HTML::TreeBuilder::XPath->new;
$xhtml->implicit_tags(1);
$xhtml->no_space_compacting(0);
$xhtml->parse_file($input)
or die("Could not parse '$file' : $!\n");
my %videos = ();
my $listpos = 0; # reset position at 0
for my $l ($xhtml->findnodes_as_strings('//a[@target="video"]/@href') ) {
my $link = qq(http://techrights.org/videos/).$l;
my $vid = basename($link);
$listpos++;
$videos{$listpos}{'link'} = $link;
$videos{$listpos}{'file'} = $vid;
}
$xhtml->destroy;
unless($is_stdin) {
close($input);
}
return(%videos);
}
sub fetch_metadata() {
my (%metadata) = (@_);
foreach my $listpos (keys %metadata) {
my @cmd = ("ffprobe",
"-v", "panic", "-of", "json", "-show_format",
$metadata{$listpos}{'link'});
open(my $json, "-|", @cmd)
or die("Could not open pipe '@cmd' : $!\m");
my @text = ();
while (my $j = <$json>) {
push (@text, $j);
}
close($json);
my $t = join("", @text);
my $d = decode_json(join("",@text));
# print Dumper($d),"\n";
if ($d->{'format'}->{'tags'}->{'title'}) {
$metadata{$listpos}{'title'} = $d->{'format'}->{'tags'}->{'title'};
}
if ($d->{'format'}->{'tags'}->{'DATE'}) {
$metadata{$listpos}{'date'} = $d->{'format'}->{'tags'}->{'DATE'};
}
# print $metadata{$listpos}{'link'},"\n";
# print $metadata{$listpos}{'file'},"\n\n";
}
return(%metadata);
}
Gemini/gemini-parse-html-to-gemini.pl
#!/usr/bin/perl
# 2021-01-11 read HTML from a file and convert it to Gemini
# not-generic, for TR only
# 2021-02-21 Proof-of-concept prototype
# not suitable for production
use utf8;
use Getopt::Std;
use File::Glob ':bsd_glob';
use HTML::TreeBuilder::XPath;
use URI;
binmode *STDOUT, ':utf8';
use English;
use warnings;
use strict;
$OUTPUT_AUTOFLUSH = 1;
our %opt;
getopts('ghmvw:', \%opt);
&usage if ($opt{'h'});
# iterate through any parmeters passed on the command line,
# treat as file names and allow globbing
my @filenames;
while (my $file = shift) {
my @files = bsd_glob($file);
foreach my $f (@files) {
push(@filenames, $f);
if($opt{'v'}) { print qq(F=$f\n); }
}
}
my $outfile = $opt{'w'} || '';
( $outfile ) = ( $outfile =~ m/^([\w\-\.\/]+)$/ );
# quit if no files were listed
&usage if($#filenames < 0);
# process each file, skipping turd files, hidden files, and temp files
while (my $infile = shift(@filenames)) {
next if ($infile =~ m/~$/);
next if ($infile =~ m/^\.\.?(?!\/)/);
next if ($infile =~ m/^#/);
my $result = &xhtml_to_gemini($infile) || exit(1);
if ($outfile) {
my ($dir) = ($outfile =~ m{^(.*/)[^/]+$});
if (! -d $dir) {
die("Directory '$dir' does not exist.\n");
}
if (! -w $dir) {
die("Directory '$dir' is not writable.\n");
}
open(my $o, '>:utf8', $outfile)
or die("Could not open '$outfile' for writing: $!\n");
if ($opt{'v'}) { print qq(Writing to "$outfile"\n); }
print $o $result;
print $o qq(\n),qq(-)x10,qq(\n);
print $o qq(=>\t/\tTechrights\n);
print $o qq(➮ Sharing is caring. );
print $o qq(Content is available under CC-BY-SA.);
close($o);
} else {
print $result;
print qq(\n),qq(-)x10,qq(\n);
print qq(=>\t/\tTechrights\n);
print qq(➮ Sharing is caring. );
print qq(Content is available under CC-BY-SA.\n);
}
}
exit(0);
sub usage {
print qq(Read pages via files or else stdin and convert them\n);
print qq(from XHTML to Gemini.\n);
$0 =~ s/^.*\///;
print qq($0: xhtml_file [xhtml_file...]\n);
print qq( -g convert internal links to Gemini\n);
print qq( -m convert internal video links, default not to\n);
print qq( -w file save output to the given file\n);
print qq( -h this help message\n);
print qq( -v increase verbosity and debugging info\n);
exit(1);
}
sub xhtml_to_gemini{
my ($file)= (@_);
# force input to be read in as UTF-8
my $input;
open ($input, "<:utf8", $file)
or die("Could not open file '$file' : error: $!\n");
# parse UTF-8
my $xhtml = HTML::TreeBuilder::XPath->new;
$xhtml->implicit_tags(1);
$xhtml->no_space_compacting(0);
$xhtml->parse_file($input)
or die("Could not parse '$file' : $!\n");
my $result = '';
if (my $l = $xhtml->findnodes_as_string('//h2/a') ) {
# this pattern is weird
if($l =~ m/>Links\D+\d+\D\d+\D\d+\D/) {
print qq(DAILY LINKS\n) if ($opt{'v'});
$result = &daily_links_page($xhtml);
} elsif($l =~ m/>Leftover Links/) {
print qq(LEFTOVER LINKS\n) if ($opt{'v'});
$result = &daily_links_page($xhtml);
} elsif($l =~ m/>Gemini Links/) {
print qq(GEMINI LINKS\n) if ($opt{'v'});
$result = &daily_links_page($xhtml);
} else {
print qq(NORMAL PAGES\n) if ($opt{'v'});
$result = &normal_page($xhtml);
}
}
$xhtml->destroy;
close($input);
return($result);
}
sub normal_page {
my ($xhtml) = (@_);
my %prefix = (
'h1' => "# ● ",
'h2' => "## ●● ",
'h3' => "### ●●● ",
'h4' => "### ●●●● ",
'h5' => "### ●●●● ",
'h6' => "### ●●●● ",
);
if ($opt{'g'}) {
$xhtml = &links2gemini($xhtml);
}
# print qq(Parsing file\n);
my $result;
for my $post ($xhtml->findnodes('//div[contains(@class,"post")]')){
# print STDERR "parsing post\n";
foreach my $hn (1 .. 5) {
# format headings
$hn = qq(h$hn);
for my $heading ($post->findnodes(".//$hn")) {
my $h = "";
if (defined($prefix{$hn})) {
$h .= $prefix{$hn};
}
$h = qq(\n\n).$h.$heading->as_text.qq(\n\n);
my $tmp = HTML::Element->new('~literal');
$tmp->push_content($h);
$heading->replace_with($tmp);
}
}
for my $pq ($post->findnodes('.//span[@class="pullQuote"]')){
my $text = $pq->as_text;
$text =~ s/\s+/ /gm;
$text =~ s/\n+/ /gm;
my $tmp = HTML::Element->new('span');
$tmp->push_content("\n\n> " . $text . "\n\n");
$pq->replace_with($tmp);
}
for my $pp ($post->findnodes('./p[@class="meta"]')) {
# keep the text but ditch the hyperlinks
my $text = $pp->as_text;
$text =~ s/\s+/ /gm;
$text =~ s/\n+/ /gm;
my $tmp = HTML::Element->new('p');
$tmp->push_content($text);
$pp->replace_with($tmp);
}
for my $ol ($post->findnodes('./ol')) {
my $item = 1;
for my $li ($ol->findnodes('./li')) {
my $href ='';
my $new_li = HTML::Element->new('~literal');
$new_li->push_content($li->as_text."\n");
for my $a ($li->findnodes('./a')) {
my $href = $a->attr('href');
$href=~ s{https?://techrights.org(/\d{1,4}/\d{1,2}/\d{1,2}/)}
{$1}x;
$href=~ s{gemini://gemini.techrights.org(/\d{1,4}/\d{1,2}/\d{1,2}/)}
{$1}x;
$href=~ s/\s/%20/g;
next if ($href=~ m/^#/);
next if ($href=~ m/\.jpe?g$/);
next if ($href=~ m/\.png$/);
next if ($href=~ m/\.gif$/);
$new_li->push_content("\n=>\t".$href."\t".
$item++." ".$a->as_text."\n");
}
$li->replace_with($new_li);
}
}
for my $ul ($post->findnodes('./ul[@class="irc-gemini"]')) {
for my $li ($ul->findnodes('./li')) {
my $href ='';
my $new_li = HTML::Element->new('li');
$new_li->push_content($li->as_text."\n");
for my $a ($li->findnodes('./a')) {
$href = $a->attr('href');
# because the direcories on Gemini and WWW are different,
# the path must change so that Gemini can point
# to gemtext instead of plain text
if( $href=~
s{^(gemini://gemini.techrights.org)/tr_text_version/}
{$1/irc-gmi/}x ) {
$href =~ s{/(irc-log[^/]+)\.txt$} {/$1.gmi}x;
$a->delete_content();
$a->push_content($href);
}
$new_li->push_content("\n=>\t".$href."\t ".
$a->as_text."\n");
}
$li->replace_with($new_li);
}
}
for my $quote ($post->findnodes('./blockquote')){
$quote->unshift_content("\n> ");
$quote->push_content("\n> ");
for my $pp ($quote->findnodes("./p")) {
$pp->unshift_content("\n> \n> ");
}
# treat pre within a blockquote as a blockquote
for my $pp ($quote->findnodes("./pre")) {
my $text = $pp->as_text;
$text =~ s/\n/\n> /gm;
my $tmp = HTML::Element->new('~literal');
$tmp->push_content($text);
$pp->replace_with($tmp);
}
}
for my $pre ($post->findnodes('./pre')){
my $text = $pre->as_text;
chomp($text);
$text = '``` ' . $text;
$text =~ s/\n/\n``` /gm;
my $tmp = HTML::Element->new('~literal');
$tmp->push_content($text.qq(\n\n));
$pre->replace_with($tmp);
}
for my $pp ($post->findnodes("./p")) {
# kludge: HTML5 video element is not recognized by parser
next unless ($pp->as_text =~ m/Video download link/
|| $pp->as_text =~ m/Reprinted/);
my @anchors=();
for my $a ($pp->findnodes('.//a[@href]')) {
my $href = $a->attr('href');
next if ($href =~ m/^#/);
$href =~ s/\s/%20/g;
my $t = $a->as_text;
if(0 && $opt{'m'}) {
# convert video link to gemini local,
# presumably it was already downloaded by now
$href =~ s{^https?://techrights.org/}
{gemini://gemini.techrights.org}x;
} else {
# don't convert the video link, leave it alone
# $href =~ s{^https?://techrights.org/} {/}x;
$t = '↺ '.$t;
}
push (@anchors, "=>\t".$href."\t".$t);
}
my $d;
if($#anchors >= 0) {
$d = $pp->as_text.qq(\n\n)
.join("\n", @anchors ).qq(\n\n);
} else {
$d = $pp->as_text.qq(\n\n);
}
my $tmp = HTML::Element->new('~literal', 'text' => $d);
$pp->replace_with($tmp);
}
for my $br ($post->findnodes("./br")) {
my $tmp = HTML::Element->new('~literal', 'text' => "\n\n");
$br->replace_with($tmp);
}
for my $pp ($post->findnodes("./p")) {
my @anchors=();
for my $a ($pp->findnodes('.//a[@href]')) {
my $href = $a->attr('href');
$href=~ s{https?://techrights.org(/\d{1,4}/\d{1,2}/\d{1,2}/)}
{$1}x;
$href=~ s{gemini://gemini.techrights.org(/\d{1,4}/\d{1,2}/\d{1,2}/)}
{$1}x;
$href=~ s/\s/%20/g;
if(my ($img) = $a->findnodes('.//img')) {
my $alt = $img->attr('alt') || 'unknown';
# lll
$a->detach_content;
$a->push_content('Image: '.$alt);
}
next if ($href=~ m/^#/);
next if ($href=~ m/\.jpe?g$/);
next if ($href=~ m/\.png$/);
next if ($href=~ m/\.gif$/);
unless($href=~m/^gemini/ || $href=~m/^\//) {
$href = $href.' ↺';
}
push (@anchors, "=>\t".$href." ".$a->as_text);
}
for my $img ($pp->findnodes('./img[@alt]')) {
my $alt = $img->attr('alt') || 'unknown';
chomp($alt);
$img->detach_content;
$img->push_content('> Image: '.$alt."\n\n");
}
my $p;
if($#anchors >= 0) {
$p = $pp->as_text.qq(\n\n). join("\n", @anchors ).qq(\n\n);
} else {
$p = $pp->as_text.qq(\n\n);
}
my $tmp = HTML::Element->new('~literal', 'text' => $p);
$pp->replace_with($tmp);
}
for my $node ($post->findnodes('./*')){
my $n = $node->as_text.qq(\n\n);
my $tmp = HTML::Element->new('~literal', 'text' => $n);
$node->replace_with($tmp);
}
# suppress warnings for $post->as_XML line
local $SIG{__WARN__} = sub {
# here we get the warning
my $warning = shift;
if ($warning =~ m{join or string at
/usr/share/perl5/HTML/Element.pm
line 1184.}x) {
return(1);
}
};
$result = $post->as_XML;
local $SIG{__WARN__};
$post->delete();
$result =~ s/<[^>]+>//g;
while ( $result =~ s/\n\n\n/\n\n/gm ) { 1; }
last;
}
$xhtml->delete;
return($result);
}
sub daily_links_page {
( my $xhtml ) = ( @_ );
# Daily Links use a difficult 1990-style structure instead of CSS :(
# they're very hard to parse
my $result = '';
if ($opt{'g'}) {
$xhtml = &links2gemini($xhtml);
}
my $tmp = HTML::Element->new('~literal');
$tmp->push_content("\n");
for my $toc ($xhtml->findnodes('//div[contains(@id,"contents")]')) {
$toc->replace_with($tmp);
}
for my $post ($xhtml->findnodes('//div[@class="post"]')){
for my $pp ($post->findnodes('./p[@class="gemini"]')) {
# delete blurb promoting gemini
$pp->delete;
}
for my $pp ($post->findnodes('./p[@class="meta"]')) {
# keep the text but ditch the hyperlinks
my $text = $pp->as_text;
$text =~ s/\s+/ /gm;
$text =~ s/\n+/ /gm;
my $tmp = HTML::Element->new('p');
$tmp->push_content($text);
$pp->replace_with($tmp);
}
for my $h1 ($post->findnodes('./h1')) {
# page date from H1
my $n = $h1->as_text.qq(\n\n);
my $tmp = HTML::Element->new('~literal');
$tmp->push_content("\n#\t● $n");
$h1->replace_with($tmp);
last;
}
for my $h2 ($post->findnodes('./h2')) {
# page title from H2
my $n = $h2->as_text.qq(\n\n);
my $tmp = HTML::Element->new('~literal');
$tmp->push_content("\n#\t● $n");
$h2->replace_with($tmp);
last;
}
for my $ul ($post->findnodes('./div/ul')) {
# deal with the nested, unorderedd lists by flattening them
$ul = &ul(0, $ul);
}
$result = $post->as_text;
$post->delete();
while ( $result =~ s/\n\n\n/\n\n/gm ) { 1; }
last;
}
return($result);
}
sub ul {
my ($depth, $ul) = ( @_ );
$depth++;
for my $h3 ($ul->findnodes('./li/h3')) {
my $new_h3 = HTML::Element->new('~literal');
$new_h3->push_content( "\n##\t".$h3->as_text."\n\n" );
$h3->replace_with($new_h3);
}
for my $u ($ul->findnodes('./li/ul')) {
$u = &ul($depth, $u);
}
my $new_li = HTML::Element->new('~literal');
for my $l ($ul->findnodes('./li[not(.//ul)]')) {
for my $h5 ($l->findnodes('./h5')) {
my $new_h5 = HTML::Element->new('~literal');
my $href ='';
my $title = $h5->as_text;
for my $a ($h5->findnodes('./a')) {
$href = $a->attr('href');
$new_li->push_content("\n=>\t".$href."\t↺ ".$title."\n\n");
}
}
for my $q ($l->findnodes('./blockquote')) {
my $new_q = HTML::Element->new('~literal');
for my $p ($q->findnodes('./p')) {
$new_li->push_content( "*\t".$p->as_text."\n\n" );
}
}
$l->replace_with($new_li);
}
return($ul);
}
sub links2gemini {
my ($xhtml) = (@_);
# convert TR http and https links to gemini
for my $anchor ($xhtml->findnodes('//a[@href]')) {
my $href = $anchor->attr('href');
if ($href!~m/^http/) {
next;
} elsif ($href=~m/\.webm$/) {
next;
}
$href =~ s{/wiki/index\.php/} {/wiki/};
my $url = URI->new($href);
# not all schemes support 'host' objects, thus the sequence matters
if (($url->scheme eq 'http' || $url->scheme eq 'https')
&& $url->host =~ m/techrights\.org$/ ) {
$url->host('gemini.techrights.org');
$url->scheme('gemini');
$anchor->attr('href' => $url->canonical);
}
}
return ($xhtml);
}
Gemini/Gemini-Proxy/gemini-proxy.pl
#!/usr/bin/perl -T
# 2202-02-04
# shell equivalent:
# echo -ne "gemini://gemini.techrights.org:1965/\r\n\r\n" \
# | openssl s_client -connect "gemini.techrights.org:1965" \
# -servername "gemini.techrights.org" -quiet -ign_eof
use 5.32.1; # state() requires 5.010 or later
# break requires 5.1.0 or later
use IO::Socket::SSL; # libio-socket-ssl-perl
# use HTML::Entities; # libhtml-parser-perl
use CGI::Fast; # libcgi-fast-perl
use CGI qw( :standard );
# use CGI::Carp qw( fatalsToBrowser ); # Remove for production use
use strict;
use warnings;
use English qw( -no_match_vars );
$CGI::POST_MAX = 1024 * 1024 * 10; # max 10MB posts
# binmode *STDOUT, ':utf8';
# binmode *STDIN, ':utf8';
my $socket_name = "/run/nginx/nginx.gemini.proxy.0.sock";
my $socket_path = "/run/nginx/";
# destroy any pre-existing socket
-S $socket_name && unlink $socket_name;
umask 007;
$ENV{FCGI_SOCKET_PATH} = $socket_name;
$ENV{FCGI_LISTEN_QUEUE} = 50;
my $url;
while (my $query = CGI::Fast->new) {
print qq(Content-type: text/html\n\n);
&print_head;
print qq(\n);
# for my $e (sort keys %ENV) {
# print qq($e : $ENV{$e}
\n);
# }
# for my $p ($query->param) {
# print qq($p ),$query->param($p),qq(
\n);
# }
my $action = $query->param('action') || 0;
my $url = $query->param('url') || 0;
$url = &sanitize($url);
if ($url) {
my ($status, @p) = ('','');
my $count = 3;
while ($count--) {
($status, @p ) = &fetch_gemini($url);
my $meta;
($status, $meta) = ($status =~ m/^([0-9]{2})\s+(.*)$/);
if ($status =~ /^3/) {
$url = &sanitize($meta);
next;
} else {
last;
}
}
my $proxy = "/proxy";
print qq(); #override alignment
print qq([Status code $status | ),&status($status), qq(]
\n);
my $http = $url;
$http =~ s|/index.gmi$|/|;
$http =~ s|(?the original in the official Web site 🢡
\n);
print qq(It may also have images and/or videos.
\n);
print qq();
}
print qq(Note: this is only a proxy (Gemini→HTTP) gateway. );
print qq(We highly recommend using proper );
print qq(Gemini client software );
print qq(instead.
\n);
&gemtext_to_xhtml($proxy, $url, @p);
} else {
&print_form;
}
&print_tail;
}
exit(0);
sub fetch_gemini {
my ($url) = (@_);
# simple client
my $cl = IO::Socket::SSL->new(
PeerHost=>"gemini.techrights.org",
PeerPort=>1965,
Timeout=>2,
Proto => 'tcp',
# SSL_ca_path => '/home/pi/Certs/',
SSL_ca_file => '/home/gemini/bin/Gemini-Proxy/Cert-Cache/gemini.tecrights.org.pem',
SSL_hostname => "gemini.techrights.org",
SSL_verify_mode => SSL_VERIFY_PEER, # overriden by next line ...
# SSL_verify_mode => SSL_VERIFY_NONE, # unsafe kludge
)
or die("Failed to connect: $!, '$SSL_ERROR'\n");
print $cl $url,"\r\n\r\n";
my @page = ();
my $status = <$cl>;
my $tmp = <$cl>;
while (my $line = <$cl> ){
push(@page,$line);
}
close($cl);
return($status, @page);
}
sub gemtext_to_xhtml {
my ($proxy, $url, @page) = (@_);
my ($path ) = ( $url =~ m{^.*//[^/]+(.*)$} );
print qq(\n);
while (my $line = shift @page) {
chomp($line);
if (! $line) {
# $line ="\x{a0}"; # non-breaking space
$line =" "; # non-breaking space
} else {
$line =~ s/\&/\&/gm;
$line =~ s/$line\n);
} elsif ($line =~ m/^##\s/) {
print qq($line
\n);
} elsif ($line =~ m/^###\s/) {
print qq($line
\n);
} elsif ($line =~ m/^=>\s/) {
my ($link, $text) = ("", "");
if (($link, $text ) = ($line =~ m/^=>\s+(\S+)\s+(.*)$/)) {
1;
} elsif (($link) = ($line =~ m/^=>\s+(\S+)/)) {
1;
} else {
0;
}
if ($link =~ m{^/} ) {
$link = "gemini://gemini.techrights.org/$link";
} elsif( $link !~ m{^\w+\://?} ) {
$link = "gemini://gemini.techrights.org/$path$link";
}
if( ! defined($text) ) {
$text = $link;
}
$link =~ s|(?$text \n);
} else {
print qq(\n);
}
} elsif ($line =~ m/^>\s/) {
$line =~ s/^>\s+//;
print qq($line
\n);
} elsif ($line =~ m/^\*\s/) {
$line =~ s/^\*\s+//;
print qq(● $line
\n);
} elsif ($line =~ m/^\`\`\`/) {
print qq(\n);
while (my $l = shift @page) {
if ($l =~ m/^\`\`\`/) {
last;
}
$l =~ s/\n);
} else {
print qq($line
\n);
}
}
print qq(\n);
return(1);
}
sub print_head {
print qq(\n);
print qq(\n);
print qq(\n);
print qq( \n);
print qq( Gemini Proxy for Techrights \n);
print qq( \n);
print qq(\n);
return(1);
}
sub print_tail {
print qq( \n\n);
return(1);
}
sub print_form {
print qq(\n);
return(1);
}
sub status {
my ($status) =(@_);
$status = int($status);
state %code = (
10 => 'INPUT',
11 => 'SENSITIVE INPUT',
20 => 'SUCCESS',
30 => 'REDIRECT - TEMPORARY',
31 => 'REDIRECT - PERMANENT',
40 => 'TEMPORARY FAILURE',
41 => 'SERVER UNAVAILABLE',
42 => 'CGI ERROR',
43 => 'PROXY ERROR',
44 => 'SLOW DOWN',
50 => 'PERMANENT FAILURE',
51 => 'NOT FOUND',
52 => 'GONE',
53 => 'PROXY REQUEST REFUSED',
59 => 'BAD REQUEST',
60 => 'CLIENT CERTIFICATE REQUIRED',
61 => 'CERTIFICATE NOT AUTHORISED',
62 => 'CERTIFICATE NOT VALID',
);
return(exists($code{$status}) ? $code{$status} : 'UNKNOWN' );
}
sub sanitize {
my ($url) = (@_);
$url =~ s/\s+//gm;
$url =~ s|/index.gmi$|/|;
$url =~ s|(?
Gemini/Gemini-Proxy/robots.txt
User-agent: * Disallow: /proxy
Gemini/Gemini-Proxy/gemini-proxy
##
# You should look at the following URL's in order to grasp a solid understanding
# of Nginx configuration files in order to fully unleash the power of Nginx.
# https://www.nginx.com/resources/wiki/start/
# https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/
# https://wiki.debian.org/Nginx/DirectoryStructure
#
# In most cases, administrators will remove this file from sites-enabled/ and
# leave it as reference inside of sites-available where it will continue to be
# updated by the nginx packaging team.
#
# This file will automatically load configuration files provided by other
# applications, such as Drupal or Wordpress. These applications will be made
# available underneath a path with that package name, such as /drupal8.
#
# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
##
# Default server configuration
#
server {
listen 80;
# listen [::]:80 default_server;
root /var/www/html;
# Add index.php to the list if you are using PHP
index index.html index.htm index.nginx-debian.html;
server_name _;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
try_files $uri $uri/ =404;
}
location /proxy {
fastcgi_pass unix:/var/run/nginx/nginx.gemini.proxy.0.sock;
fastcgi_param SCRIPT_FILENAME /usr/local/bin/gemini-proxy.pl;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
include fastcgi_params;
deny 20.34.0.0/15; # bing 病
deny 20.36.0.0/14; # bing 病
deny 20.40.0.0/13; # bing 病
deny 20.48.0.0/12; # bing 病
deny 20.64.0.0/10; # bing 病
deny 20.128.0.0/16; # bing 病
deny 23.96.0.0/13; # bing 病
deny 34.64.0.0/10; # googlebot
deny 40.74.0.0/15; # bing 病
deny 40.76.0.0/14; # bing 病
deny 40.80.0.0/12; # bing 病
deny 40.96.0.0/12; # bing 病
deny 40.124.0.0/16; # bing 病
deny 40.112.0.0/13; # bing 病
deny 40.120.0.0/14; # bing 病
deny 40.125.0.0/17; # bing 病
deny 52.84.0.0/14; # bing 病
deny 52.88.0.0/13; # bing 病
deny 52.136.0.0/13; # ddg / bing 病
deny 52.132.0.0/14; # ddg / bing 病
deny 52.145.0.0/16; # bing 病
deny 52.146.0.0/15; # bing 病
deny 52.148.0.0/14; # bing 病
deny 52.152.0.0/13; # bing 病
deny 52.160.0.0/11; # bing 病
deny 157.56.0.0/14; # bing 病
deny 157.54.0.0/15; # bing 病
deny 157.60.0.0/16; # bing 病
deny 207.46.0.0/16; # bing 病
deny 37.252.64.0/19;
deny 66.249.64.0/19; # google bot
deny 116.128.0.0/10; # baidu
deny 162.142.125.0/24; # censys
deny 185.250.240.0/24;
deny 208.80.192.0/21;
deny 77.88.0.0/18; # yandex
deny 93.158.128.0/18; # yandex
}
location = / {
return 301 http://gemini.techrights.org/proxy?url=gemini://gemini.techrights.org/;
}
}
server {
listen 443 ssl;
# listen [::]:443 ssl;
include snippets/self-signed.conf;
include snippets/ssl-params.conf;
root /var/www/html;
# Add index.php to the list if you are using PHP
index index.html index.htm index.nginx-debian.html;
server_name _;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
try_files $uri $uri/ =404;
}
location /proxy {
fastcgi_pass unix:/var/run/nginx/nginx.gemini.proxy.0.sock;
fastcgi_param SCRIPT_FILENAME /usr/local/bin/gemini-proxy.pl;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
include fastcgi_params;
}
location = / {
return 301 https://gemini.techrights.org/proxy?url=gemini://gemini.techrights.org/;
}
}
# Virtual Host configuration for example.com
#
# You can move that to a different file under sites-available/ and symlink that
# to sites-enabled/ to enable it.
#
#server {
# listen 80;
# listen [::]:80;
#
# server_name example.com;
#
# root /var/www/example.com;
# index index.html;
#
# location / {
# try_files $uri $uri/ =404;
# }
#}
Gemini/Gemini-Proxy/Cert-Cache/.directory-listing-ok
Gemini/Gemini-Proxy/Cert-Cache/gemini.tecrights.org.pem
-----BEGIN CERTIFICATE----- MIIBVTCB+6ADAgECAgEqMAoGCCqGSM49BAMCMCAxHjAcBgNVBAMMFWdlbWluaS50 ZWNocmlnaHRzLm9yZzAgFw03NTAxMDEwMDAwMDBaGA80MDk2MDEwMTAwMDAwMFow IDEeMBwGA1UEAwwVZ2VtaW5pLnRlY2hyaWdodHMub3JnMFkwEwYHKoZIzj0CAQYI KoZIzj0DAQcDQgAEI0o5Lnz2dDrRtj/G7fbTQQlIij6iJxkw6B6pFd19Bk+PoPxy sVjC81ORc8h14HbDD8NBDWIL0SdPtzcYNT/A+aMkMCIwIAYDVR0RBBkwF4IVZ2Vt aW5pLnRlY2hyaWdodHMub3JnMAoGCCqGSM49BAMCA0kAMEYCIQDCs7jcvilZocLT lsvT9algZT2z6s1UHJ2VPffKIh2B2wIhAOuiE2nhxYijTimG+YM4xMH4bNIKA6Be wJ8hAedbybAt -----END CERTIFICATE-----
Gemini/Gemini-Proxy/.directory-listing-ok
Gemini/Gemini-Proxy/gemini-proxy.service
[Unit] Description=Gemini Proxy Service After=network.target After=nginx.target [Service] Type=simple User=gemprox PermissionsStartOnly=true ExecStartPre=-/bin/mkdir -p /run/nginx/ ExecStartPre=/bin/chmod g=rwxs /run/nginx/ ExecStartPre=/bin/chown www-data:gemprox /run/nginx/ ExecStart=/usr/local/bin/gemini-proxy.pl # or always, on-abort, etc Restart=on-failure Nice=5 [Install] WantedBy=multi-user.target
Gemini/tr-update-planet.sh
#!/bin/bash
# Initial 2022-03-06
# Last updated 2022-03-18
set -v -x
# PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin
# Get key dates first
daybeforeyesterday=$(date --date "2 days ago" +"%Y-%m-%d")
yesterday=$(date --date yesterday +"%Y-%m-%d")
today=$(date --date today +"%Y-%m-%d")
##################################
# Start main page of Planet
echo "# Planet Gemini" > ~/gemini/planet/index.gmi
echo "Welcome to Planet Gemini. This is where we aggregate sources to help us identify new material in gopher:// and gemini:// space." >> ~/gemini/planet/index.gmi
echo "## Get added" >> ~/gemini/planet/index.gmi
echo "Contact us if you want yours to be added too." >> ~/gemini/planet/index.gmi
echo "=> /about Contacting us" >> ~/gemini/planet/index.gmi
echo "=> / Go back to Techrights (Main Index)" >> ~/gemini/planet/index.gmi
echo "# New/Updated So Far Today and Yesterday" >> ~/gemini/planet/index.gmi
echo "=> /planet/latest Latest Items" >> ~/gemini/planet/index.gmi
##################################
# Update the page with latest posts in the Planet (to use prior snapshot, not upcoming)
echo "# Latest Posts/Articles Across Geminispace" > ~/gemini/planet/latest/index.gmi
echo "This page aggregates posts found for the current day and prior day. It's typically updated every 3 hours." >> ~/gemini/planet/latest/index.gmi
cat ~/gemini/planet/othercapsules/${today}.gmi >> ~/gemini/planet/latest/index.gmi
cat ~/gemini/planet/othercapsules/${yesterday}.gmi >> ~/gemini/planet/latest/index.gmi
echo "____________________________________________________" >> ~/gemini/planet/latest/index.gmi
echo "=> / Go back to Techrights (Main Index)" >> ~/gemini/planet/latest/index.gmi
echo "=> /planet Planet (Full)" >> ~/gemini/planet/latest/index.gmi
echo -n "Last update cycle ended " >> ~/gemini/planet/latest/index.gmi
date --date today "+%A, %B %d, %Y at %H:%M" >> ~/gemini/planet/latest/index.gmi
##################################
# Now go back to main Planet page
echo "## More Dates" >> ~/gemini/planet/index.gmi
echo "=> /planet/othercapsules Older Dates (Archive)" >> ~/gemini/planet/index.gmi
echo "=> /planet/othercapsules/allinone.gmi All in one, past 3 days combined, fused and compacted" >> ~/gemini/planet/index.gmi
echo "=> /planet/othercapsules/new.gmi New (added since last update cycle)" >> ~/gemini/planet/index.gmi
echo "## Updates" >> ~/gemini/planet/index.gmi
echo -n "Last update cycle started " >> ~/gemini/planet/index.gmi
date --date today "+%A, %B %d, %Y at %H:%M" >> ~/gemini/planet/index.gmi
echo "Page updated every 3 hours ⦼" >> ~/gemini/planet/index.gmi
echo "## Sources" >> ~/gemini/planet/index.gmi
echo "Some suggested feeds (for syndication) are listed below." >> ~/gemini/planet/index.gmi
echo "=> /git/tr-git/Gemini/gemini-feeds.gmi List of feeds" >> ~/gemini/planet/index.gmi
echo "=> /git/tr-git/Gemini/gemini-capcom.gmi More active feeds" >> ~/gemini/planet/index.gmi
echo "# Full Lists (Item Listing, Undated)" >> ~/gemini/planet/index.gmi
echo "Pertinent capsules followed by list of lists, credit added where it is due" >> ~/gemini/planet/index.gmi
# Note: dirty hacks below, might be worth tidying up in the future
# echo -ne "gemini://gemini.techrights.org/feed.xml\r\n\r\n" | openssl s_client -servername gemini.techrights.org -quiet -ign_eof -connect gemini.techrights.org:1965 | sed -e '1{/^[0-9][0-9]/d}' | xmlstarlet sel -E utf-8 -T -t -m '//item' -o '=> ' -v './link' -o " " -v './title' -n >> ~/gemini/planet/index.gmi
echo "## mieum" >> ~/gemini/planet/index.gmi
(echo "gemini://rawtext.club/~mieum/poems/atom.xml"; sleep 1) \
| openssl s_client -connect rawtext.club:1965 \
| grep -e title -e link \
| tac \
| sed "s/ //" \
| sed "s/<\/title>//" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" >> ~/gemini/planet/index.gmi
echo "## 🅰🆅🅰🅻🅾🆂.🅼🅴" >> ~/gemini/planet/index.gmi
echo -ne "gemini://avalos.me/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername avalos.me -quiet -ign_eof -connect avalos.me:1965 \
| grep -e title -e link \
| tac \
| sed "s//=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s///" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🆂🅸🆇🅾🅷🆃🅷🆁🅴🅴.🅲🅾🅼" >> ~/gemini/planet/index.gmi
echo -ne "gemini://sixohthree.com:1965/atom.xml\r\n\r\n" \
| openssl s_client -servername sixohthree.com -quiet -ign_eof -connect sixohthree.com:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f4-4 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🆅🅺3.🆆🆃🅵 " >> ~/gemini/planet/index.gmi
echo -ne "gemini://vk3.wtf:1965/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername vk3.wtf -quiet -ign_eof -connect vk3.wtf:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🆃🅷🅴🅻🅰🅼🅱🅳🅰🅻🅰🅱.🆇🆈🆉 " >> ~/gemini/planet/index.gmi
echo -ne "gemini://thelambdalab.xyz:1965/atom.xml\r\n\r\n" \
| openssl s_client -servername thelambdalab.xyz -quiet -ign_eof -connect thelambdalab.xyz:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".txt" >> ~/gemini/planet/index.gmi
# OFFLINE JULY 2022:
# echo "## 🅳🆁🅴🆆🅳🅴🆅🅰🆄🅻🆃.🅲🅾🅼" >> ~/gemini/planet/index.gmi
# echo -ne "gemini://drewdevault.com/feed.xml\r\n\r\n" \
# | openssl s_client -servername drewdevault.com -quiet -ign_eof -connect drewdevault.com:1965 \
# | grep -e title -e link \
# | tac \
# | sed "s/<link>/=> /" \
# | cut --d="\"" -f2-2 \
# | sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
# | sed "s/ //" \
# | sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
# | grep "=>" \
# | grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## Tux Machines">> ~/gemini/planet/index.gmi
echo -ne "gemini://gemini.tuxmachines.org/feed.xml\r\n\r\n" \
| openssl s_client -servername gemini.tuxmachines.org -quiet -ign_eof -connect gemini.tuxmachines.org:1965 \
| grep -e title -e link \
| sed "s/<link>/=> /" \
| cut --d="\"" -f4-4 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅼🅾🅳🅳🅴🅳🅱🅴🅰🆁.🆇🆈🆉">> ~/gemini/planet/index.gmi
echo -ne "gemini://moddedbear.xyz/logs/atom.xml\r\n\r\n" \
| openssl s_client -servername moddedbear.xyz -quiet -ign_eof -connect moddedbear.xyz:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/ <title>//" -e "s/<\/title>//; s/<\/link>//" \
| sed "s/ //" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
# echo "## nader.pm">> ~/gemini/planet/index.gmi
# echo -ne "gemini://nader.pm/atom.xml\r\n\r\n" | openssl s_client -servername nader.pm -quiet -ign_eof -connect nader.pm:1965 | grep -e title -e link | tac | sed "s/<link>/=> /" | cut --d="\"" -f2-2 | sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" | sed "s/gemini:/=> gemini:/" | sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' | grep "=>" | grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅶🅴🅼🅸🅽🅸.🅸🅾🆂🅰.🅸🆃">> ~/gemini/planet/index.gmi
echo -ne "gemini://gemini.iosa.it/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername gemini.iosa.it -quiet -ign_eof -connect gemini.iosa.it:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅲🅰🅻🅲🆄🅾🅳🅴.🅲🅾🅼">> ~/gemini/planet/index.gmi
echo -ne "gemini://calcuode.com/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername calcuode.com -quiet -ign_eof -connect calcuode.com:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
### Inactive May 2022, hence commented out
### echo "## jfh.me">> ~/gemini/planet/index.gmi
### echo -ne "gemini://jfh.me/atom.xml\r\n\r\n" \
### | openssl s_client -servername jfh.me -quiet -ign_eof -connect jfh.me:1965 \
### | grep -e title -e link \
### | tac \
### | sed "s/<link>/=> /" \
### | cut --d="\"" -f2-2 \
### | sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
### | sed "s/gemini:/=> gemini:/" \
### | sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
### | grep "=>" \
### | grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅶🅰🅻🅰🆇🆈🅷🆄🅱.🆄🅺">> ~/gemini/planet/index.gmi
echo -ne "gemini://galaxyhub.uk/articles/atom.xml\r\n\r\n" \
| openssl s_client -servername galaxyhub.uk -quiet -ign_eof -connect galaxyhub.uk:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
# echo "## gemini://cosmic.voyage">> ~/gemini/planet/index.gmi
# echo -ne "gemini://cosmic.voyage/atom.xml\r\n\r\n" | openssl s_client -servername cosmic.voyage -quiet -ign_eof -connect cosmic.voyage:1965 | grep -e title -e link | tac | sed "s/<link>/=> /" | cut --d="\"" -f2-2 | sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" | sed "s/gemini:/=> gemini:/" | sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' | grep "=>" | grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅶🅴🅼🅸🅽🅸.🆄🅲🅰🅽🆃.🅾🆁🅶">> ~/gemini/planet/index.gmi
echo -ne "gemini://gemini.ucant.org/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername gemini.ucant.org -quiet -ign_eof -connect gemini.ucant.org:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅻🅾🆃🆃🅰🅻🅸🅽🆄🆇🅻🅸🅽🅺🆂.🅲🅾🅼">> ~/gemini/planet/index.gmi
echo -ne "gemini://gemini.lottalinuxlinks.com/posts/feed.xml\r\n\r\n" \
| openssl s_client -servername gemini.lottalinuxlinks.com -quiet -ign_eof -connect gemini.lottalinuxlinks.com:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep -v "id>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅰🅿.🆂🆆🅰🅽.🆀🆄🅴🆂🆃">> ~/gemini/planet/index.gmi
echo -ne "gemini://cap.swan.quest/p/atom.xml\r\n\r\n" \
| openssl s_client -servername cap.swan.quest -quiet -ign_eof -connect cap.swan.quest:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
echo "## 🅽🆈🆃🅿🆄.🅲🅾🅼" >> ~/gemini/planet/index.gmi
echo -ne "gemini://nytpu.com:1965/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername nytpu.com -quiet -ign_eof -connect nytpu.com:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="'" -f4-4 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/\/\//=> gemini:\/\//" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
##################################
# Start imports section/s
echo "# Imported by nytpu.com">> ~/gemini/planet/index.gmi
echo "Credit due (original):" >> ~/gemini/planet/index.gmi
echo "=> gemini://nytpu.com/feed.gmi Alex" >> ~/gemini/planet/index.gmi
echo -ne "gemini://nytpu.com/feed.gmi\r\n\r\n" \
| openssl s_client -servername nytpu.com -quiet -ign_eof -connect nytpu.com:1965 \
| sed -n '/##/,$p' >> ~/gemini/planet/index.gmi
##################################
# Add/chain CAPCOM
echo "# Imported via CAPCOM">> ~/gemini/planet/index.gmi
echo "Credit due (original):" >> ~/gemini/planet/index.gmi
echo "=> gemini://gemini.circumlunar.space/capcom/ CAPCOM" >> ~/gemini/planet/index.gmi
echo -ne "gemini://gemini.circumlunar.space/capcom/\r\n\r\n" \
| openssl s_client -servername gemini.circumlunar.space -quiet -ign_eof -connect gemini.circumlunar.space:1965 \
| sed -n '/Aggregating 100/,$p' >> ~/gemini/planet/index.gmi
##################################
# Add/chain Spacewalk
echo "# Imported via Spacewalk">> ~/gemini/planet/index.gmi
echo "Credit due (original):" >> ~/gemini/planet/index.gmi
echo "=> gemini://rawtext.club/~sloum/spacewalk.gmi Spacewalk" >> ~/gemini/planet/index.gmi
# echo "\`\`\`" >> ~/gemini/planet/index.gmi
echo -ne "gemini://rawtext.club/~sloum/spacewalk.gmi\r\n\r\n" \
| openssl s_client -servername rawtext.club -quiet -ign_eof -connect rawtext.club:1965 \
| sed -n '/Oldest/,$p' >> ~/gemini/planet/index.gmi
##################################
# Add/chain Skyjake
echo "# Imported via Skyjake">> ~/gemini/planet/index.gmi
echo "Credit due (original):" >> ~/gemini/planet/index.gmi
echo "=> gemini://skyjake.fi/~Cosmos/ Skyjake" >> ~/gemini/planet/index.gmi
# openssl s_client -quiet -crlf \
# -servername skyjake.fi \
# -connect skyjake.fi:1965 \
# | awk '{ print "response: " $0 }' gemini://skyjake.fi/~Cosmos/ |
# openssl s_client -quiet -crlf \
# -servername skyjake.fi \
# -connect skyjake.fi:1965 \
#| awk '{ print "response: " $0 }'
#gemini://skyjake.fi/~Cosmos/ | sed -n '/202/,$p' >> ~/gemini/planet/index.gmi
# Temporary workaround below because the capsule acts weird
mv ~/Downloads/~Cosmos.gmi /tmp/cosmos
cat /tmp/cosmos | sed -n '/202/,$p' >> ~/gemini/planet/index.gmi
# sh ~/bin/gemini-get-cosmo.sh
##################################
# Add/chain Warmedal
echo "# Imported via Warmedal">> ~/gemini/planet/index.gmi
echo "Credit due (original):" >> ~/gemini/planet/index.gmi
echo "=> gemini://warmedal.se/~antenna/index.gmi Warmedal" >> ~/gemini/planet/index.gmi
echo "" >> ~/gemini/planet/index.gmi
echo -ne "gemini://warmedal.se/~antenna/index.gmi\r\n\r\n" \
| openssl s_client -servername warmedal.se -quiet -ign_eof -connect warmedal.se:1965 \
| sed -n '/202/,$p' >> ~/gemini/planet/index.gmi
##################################
# Add/chain Calcuode
echo "# Imported via Calcuode">> ~/gemini/planet/index.gmi
echo "Credit due (original):" >> ~/gemini/planet/index.gmi
echo "=> gemini://calcuode.com/gmisub-aggregate.gmi Calcuode" >> ~/gemini/planet/index.gmi
echo -ne "gemini://calcuode.com/gmisub-aggregate.gmi\r\n\r\n" \
| openssl s_client -servername calcuode.com -quiet -ign_eof -connect calcuode.com:1965 \
| sed -n '/Entries/,$p' >> ~/gemini/planet/index.gmi
echo '________________________________________________' >> ~/gemini/planet/index.gmi
echo -n 'Last updated ' >> ~/gemini/planet/index.gmi
date --date today "+%A, %B %d, %Y at %H:%M" >> ~/gemini/planet/index.gmi
echo "=> / Go back to Techrights (Main Index)" >> ~/gemini/planet/index.gmi
##################################
# Now update the archives
# We limits the scope of the scanner to imported sections
echo -n "## News for " > ~/gemini/planet/othercapsules/${yesterday}.gmi
echo ${yesterday} >> ~/gemini/planet/othercapsules/${yesterday}.gmi
sed -n '/# Imported/,$p' ~/gemini/planet/index.gmi \
| sed -n -e "/${yesterday}/,/${daybeforeyesterday}/p" \
| sort -u -k2,2 \
| grep -v " 2020-" \
| grep -v " 2021-" \
| sort -k3,3 \
| grep "=>" >> ~/gemini/planet/othercapsules/${yesterday}.gmi
# Consider adding sed 's/gopher:\/\//gopher:\/\/ [Gopher Link]/' or similar if many gopher links exist or need culling
echo -n "## News for " > ~/gemini/planet/othercapsules/${today}.gmi
echo ${today} >> ~/gemini/planet/othercapsules/${today}.gmi
sed -n '/# Imported/,$p' ~/gemini/planet/index.gmi \
| sed -n -e "/${today}/,/${yesterday}/p" \
| sort -u -k2,2 \
| grep -v " 2020-" \
| grep -v " 2021-" \
| sort -k3,3 \
| grep "=>" >> ~/gemini/planet/othercapsules/${today}.gmi
# All in one, past 3 days combined, fused and compacted
cp ~/gemini/planet/othercapsules/allinone.gmi ~/gemini/planet/othercapsules/allinone-last.gmi
echo "# 3-day Outline for Geminispace" > ~/gemini/planet/othercapsules/allinone.gmi
cat ~/gemini/planet/othercapsules/${today}.gmi \
~/gemini/planet/othercapsules/${yesterday}.gmi \
~/gemini/planet/othercapsules/${daybeforeyesterday}.gmi \
| uniq \
| grep -v "News for 20" \
| sort -u -k2,2 \
| sort -r -k3,3 >> ~/gemini/planet/othercapsules/allinone.gmi
echo '____________‿︵‿︵ʚ˚̣̣̣͙ɞ・❉・ ʚ˚̣̣̣͙ɞ‿︵‿︵_________' >> ~/gemini/planet/othercapsules/allinone.gmi
echo -n 'Last updated ' >> ~/gemini/planet/othercapsules/allinone.gmi
date --date today "+%A, %B %d, %Y at %H:%M" >> ~/gemini/planet/othercapsules/allinone.gmi
echo "=> /planet/othercapsules/new.gmi Latest updates, sorted by time" >> ~/gemini/planet/othercapsules/allinone.gmi
echo "=> / Go back to Techrights (Main Index)" >> ~/gemini/planet/othercapsules/allinone.gmi
# Start the page afresh at first 3 hours of the day (assumes update cycles no greater than
# 3 hours apart
case $(date +%H:%M) in
(0[0]:*) echo "# New in Gemini (Since Last Time or Update Cycle)" > ~/gemini/planet/othercapsules/new.gmi;;
(*) echo " ‿︵•‿︵•‿︵•‿︵" >> ~/gemini/planet/othercapsules/new.gmi;;
esac
# Show differences in the direction of the "new" version, annul the "old" one aging away
diff ~/gemini/planet/othercapsules/allinone.gmi ~/gemini/planet/othercapsules/allinone-last.gmi \
| egrep '^[d<]' \
| cut -c3-256 >> ~/gemini/planet/othercapsules/new.gmi
# Start assembling the archives
echo "# Gemini Archive" > ~/gemini/planet/othercapsules/index.gmi
echo "Select any of the dates below. Those show all the Geminispace updates visible to us." >> ~/gemini/planet/othercapsules/index.gmi
# Process starts of month specially
find /home/gemini/gemini/planet/othercapsules/ \
-type f \
-name "2*" \
-exec basename {} \; \
| sort \
| sed -r -e 's/([0-9]{4}-[0-9]{2}-[0-9]{2})\.gmi/\1.gmi \1/; s/^/=> /' \
-e '/01-01.*/i ### January ㋀' \
-e '/02-01.*/i ### February ㋁' \
-e '/03-01.*/i ### March ㋂' \
-e '/04-01.*/i ### April ㋃' \
-e '/05-01.*/i ### May ㋄' \
-e '/06-01.*/i ### June ㋅' \
-e '/07-01.*/i ### July ㋆' \
-e '/08-01.*/i ### August ㋇' \
-e '/09-01.*/i ### September ㋈' \
-e '/10-01.*/i ### October ㋉' \
-e '/11-01.*/i ### November ㋊' \
-e '/12-01.*/i ### December ㋋' \
>> ~/gemini/planet/othercapsules/index.gmi
echo -n 'Last updated ' >> ~/gemini/planet/othercapsules/index.gmi
date --date today "+%A, %B %d, %Y at %H:%M" >> ~/gemini/planet/othercapsules/index.gmi
echo "=> / Go back to Techrights (Main Index)" >> ~/gemini/planet/othercapsules/index.gmi
# Finish gracefully
exit 0
##################################
# Note/stale below
# -------
# echo -ne "gemini://gemini.jayeless.net/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername gemini.jayeless.net -quiet -ign_eof -connect gemini.jayeless.net:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep ".gmi" >> ~/gemini/planet/index.gmi
# echo -ne "gemini://gemini.jayeless.net/gemlog/atom.xml\r\n\r\n" \
| openssl s_client -servername gemini.jayeless.net -quiet -ign_eof -connect gemini.jayeless.net:1965 \
| sed -e '1{/^[0-9][0-9]/d}' \
| xmlstarlet sel -E utf-8 -T -t -m '//item' -o '=> ' -v './link' -o " " -v './title' -n
# ------
# echo -ne "gemini://midnight.pub/feed.xml\r\n\r\n" \
| openssl s_client -servername midnight.pub -quiet -ign_eof -connect midnight.pub:1965 \
| grep -e title -e link \
| tac \
| sed "s/<link>/=> /" \
| cut --d="\"" -f2-2 \
| sed -e "s/<title>//" -e "s/<\/title>//" -e "s/<\/link>//" \
| sed "s/gemini:/=> gemini:/" \
| sed -e :a -e N -e '$!ba' -e 's/gmi\n/gmi /g' \
| grep "=>" \
| grep "posts"
# echo -ne "gemini://midnight.pub/feed.xml\r\n\r\n" \
| openssl s_client -servername midnight.pub -quiet -ign_eof -connect midnight.pub:1965 \
| sed -e '1{/^[0-9][0-9]/d}' \
| xmlstarlet sel -E utf-8 -T -t -m '//item' -o '=> ' -v './link' -o " " -v './title' -n
# ------
</pre>
<div><a href="https://gemini.nicfab.eu/lxy/external/gemini.techrights.org%2F">Back to main index</a></div>
</main>
</body>
</html>