AuraGem Servers > Commit [93b5ba6]
Mon May 20, 2024 11:02 AM -0500
Aggregator tool, UDC stuff .gitignore | 1 + aggregator_tool/main.go | 287 +++++++++++++++++++++++++++++++++++++++++++++++++++++ gemini/gemini.go | 7 +++++++ gemini/github.go | 5 +++++ gemini/music/music.go | 38 +++++++++++++++++++------------------- gemini/music/radio.go | 5 +++-- gemini/music/stations.go | 6 +++--- gemini/search/db.go | 40 ++++++++++++++++++++-------------------- gemini/search/search.go | 284 +++++++++++++++++++++++++++++++++++++---------------- gemini/search/types.go | 1 + gemini/starwars/starwars.go | 24 ++++++++++++------------ gemini/texts/christianity/christianity.go | 8 +++++++- gemini/texts/islam/islam.go | 5 ++++- gemini/texts/judaism/judaism.go | 3 +++ gemini/texts/texts.go | 1 + gemini/weather.go | 2 +- gemini/youtube/youtube.go | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 4 ++-- go.sum | 10 ++++++++++ migration/migrations/2021-06-08T140237Z_SearchInitial.go | 1 + stats_tool/main.go | 5 +++++
Commit Hash: 93b5ba6f989a7b732c8b6959b7356a3b91446f3a
Tree Hash: 8cb11cfd752a0f3ef1d331cea2724e9162d5654b
Date: 2024-05-20T11:02:46-05:00
Changes
.gitignore
... | ...
8 | data/
9 | access.log
10 | *.pem
11 | *.key
12 | *.crt
13 | *.exe
14 | auragem_sis
15 | config/config.go
16 | main
17 | main.backup
18 | SIS/
19 | *.prof
+ 13 | aggregator_tool/aggregator_tool
aggregator_tool/main.go (created)
+ 1 | package main
+ 2 |
+ 3 | import (
+ 4 | "context"
+ 5 | "database/sql"
+ 6 | "fmt"
+ 7 | "os"
+ 8 | "path/filepath"
+ 9 | "strings"
+ 10 | "time"
+ 11 |
+ 12 | "gitlab.com/clseibold/auragem_sis/db"
+ 13 | "gitlab.com/clseibold/auragem_sis/gemini/search"
+ 14 | "golang.org/x/text/language"
+ 15 | )
+ 16 |
+ 17 | // Aggregator tool queries the database to construct pages for the aggregator.
+ 18 |
+ 19 | func main() {
+ 20 | conn := db.NewConn(db.SearchDB)
+ 21 |
+ 22 | page := 1
+ 23 | for {
+ 24 | hasNext := getPage("/home/clseibold/ServerData/auragem_sis/SIS/auragem_gemini/search/yearposts/", page, conn)
+ 25 | if !hasNext {
+ 26 | break
+ 27 | }
+ 28 | page++
+ 29 | }
+ 30 | }
+ 31 |
+ 32 | func getPage(root string, page int, conn *sql.DB) bool {
+ 33 | results := 40
+ 34 | skip := (page - 1) * results
+ 35 |
+ 36 | pages, totalResultsCount := getPagesWithPublishDateFromLastYear(conn, results, skip)
+ 37 |
+ 38 | resultsStart := skip + 1
+ 39 | resultsEnd := search.Min(totalResultsCount, skip+results) // + 1 - 1
+ 40 | hasNextPage := resultsEnd < totalResultsCount && totalResultsCount != 0
+ 41 | hasPrevPage := resultsStart > results
+ 42 |
+ 43 | var builder strings.Builder
+ 44 | buildPageResults(&builder, pages, false, false)
+ 45 |
+ 46 | if hasPrevPage {
+ 47 | if page-1 <= 1 {
+ 48 | fmt.Fprintf(&builder, "\n=> /search/yearposts/ Previous Page\n")
+ 49 | } else {
+ 50 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d.gmi Previous Page\n", page-1)
+ 51 | }
+ 52 | }
+ 53 | if hasNextPage && !hasPrevPage {
+ 54 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d.gmi Next Page\n", page+1)
+ 55 | } else if hasNextPage && hasPrevPage {
+ 56 | fmt.Fprintf(&builder, "=> /search/yearposts/%d.gmi Next Page\n", page+1)
+ 57 | }
+ 58 |
+ 59 | doc := fmt.Sprintf(`# Publications From The Past Year
+ 60 |
+ 61 | => /search/ Home
+ 62 | => /search/s/ Search
+ 63 |
+ 64 | Note: Currently lists only English posts.
+ 65 |
+ 66 | %s
+ 67 | `, builder.String())
+ 68 |
+ 69 | filename := filepath.Join(root, "index.gmi")
+ 70 | if page > 1 {
+ 71 | filename = filepath.Join(root, fmt.Sprintf("%d.gmi", page))
+ 72 | }
+ 73 | err := os.WriteFile(filename, []byte(doc), 0600)
+ 74 | if err != nil {
+ 75 | panic(err)
+ 76 | }
+ 77 |
+ 78 | return hasNextPage
+ 79 | }
+ 80 |
+ 81 | // TODO: Allow for different languages
+ 82 | // NOTE: Blank language fields are considered English
+ 83 | func getPagesWithPublishDateFromLastYear(conn *sql.DB, results int, skip int) ([]search.Page, int) {
+ 84 | query := fmt.Sprintf("SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE publishdate > dateadd(-1 year to ?) AND publishdate < dateadd(2 day to ?) AND (language = '' OR language LIKE 'en%%' OR language LIKE 'EN%%' OR language LIKE 'eng%%' OR language LIKE 'ENG%%') AND scheme <> 'scroll' AND hidden = false AND domainid <> 9 ORDER BY publishdate DESC", results, skip)
+ 85 | rows, rows_err := conn.QueryContext(context.Background(), query, time.Now().UTC(), time.Now().UTC())
+ 86 |
+ 87 | var pages []search.Page = make([]search.Page, 0, results)
+ 88 | var totalCount int
+ 89 | if rows_err == nil {
+ 90 | defer rows.Close()
+ 91 | for rows.Next() {
+ 92 | var page search.Page
+ 93 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 94 | if scan_err == nil {
+ 95 | pages = append(pages, page)
+ 96 | } else {
+ 97 | prevPage := search.Page{}
+ 98 | if len(pages) > 0 {
+ 99 | prevPage = pages[len(pages)-1]
+ 100 | }
+ 101 | panic(fmt.Errorf("scan error after page %v; %s", prevPage, scan_err.Error()))
+ 102 | }
+ 103 | }
+ 104 |
+ 105 | if err := rows.Err(); err != nil {
+ 106 | panic(err)
+ 107 | }
+ 108 | }
+ 109 |
+ 110 | return pages, totalCount
+ 111 | }
+ 112 |
+ 113 | func buildPageResults(builder *strings.Builder, pages []search.Page, useHighlight bool, showScores bool) {
+ 114 | for _, page := range pages {
+ 115 | typeText := ""
+ 116 | if page.Prompt != "" {
+ 117 | typeText = "Input Prompt • "
+ 118 | } else if page.Feed {
+ 119 | typeText = "Gemsub Feed • "
+ 120 | }
+ 121 |
+ 122 | publishDateString := ""
+ 123 | if page.PublishDate.Year() > 1800 && page.PublishDate.Year() <= time.Now().Year() {
+ 124 | publishDateString = fmt.Sprintf("Published on %s • ", page.PublishDate.Format("2006-01-02"))
+ 125 | }
+ 126 |
+ 127 | artist := ""
+ 128 | if page.AlbumArtist != "" {
+ 129 | artist = "by " + page.AlbumArtist + " • "
+ 130 | } else if page.Artist != "" {
+ 131 | artist = "by " + page.Artist + " • "
+ 132 | }
+ 133 |
+ 134 | langText := ""
+ 135 | if page.Content_type == "text/gemini" || page.Content_type == "" || strings.HasPrefix(page.Content_type, "text/") {
+ 136 | // NOTE: This will just get the first language listed. In the future, list all languages by splitting on commas
+ 137 | tag, _ := language.MatchStrings(languageMatcher, page.Language)
+ 138 | str := langTagToText(tag)
+ 139 | if str != "" {
+ 140 | langText = fmt.Sprintf("%s • ", str)
+ 141 | }
+ 142 | }
+ 143 |
+ 144 | size := float64(page.Size)
+ 145 | sizeLabel := "B"
+ 146 | if size > 1024 {
+ 147 | size /= 1024.0
+ 148 | sizeLabel = "KB"
+ 149 | }
+ 150 | if size > 1024 {
+ 151 | size /= 1024.0
+ 152 | sizeLabel = "MB"
+ 153 | }
+ 154 | if size > 1024 {
+ 155 | size /= 1024.0
+ 156 | sizeLabel = "GB"
+ 157 | }
+ 158 |
+ 159 | score := ""
+ 160 | if showScores {
+ 161 | score = fmt.Sprintf(" (Score: %f)", page.Score)
+ 162 | }
+ 163 |
+ 164 | if page.Title == "" {
+ 165 | fmt.Fprintf(builder, "=> %s %s%s\n", page.Url, page.Url, score)
+ 166 | fmt.Fprintf(builder, "%s%s%s%s%d Lines • %.1f %s\n", typeText, publishDateString, langText, artist, page.Linecount, size, sizeLabel)
+ 167 | } else {
+ 168 | fmt.Fprintf(builder, "=> %s %s%s\n", page.Url, page.Title, score)
+ 169 | fmt.Fprintf(builder, "%s%s%s%s%d Lines • %.1f %s • %s\n", typeText, publishDateString, langText, artist, page.Linecount, size, sizeLabel, page.Url)
+ 170 | }
+ 171 | if useHighlight {
+ 172 | fmt.Fprintf(builder, "> %s\n", page.Highlight)
+ 173 | }
+ 174 | fmt.Fprintf(builder, "\n")
+ 175 | }
+ 176 | }
+ 177 |
+ 178 | // ----- Language Stuff -----
+ 179 |
+ 180 | var Esperanto language.Tag = language.MustParse("eo")
+ 181 | var Yiddish language.Tag = language.MustParse("yi")
+ 182 | var AustralianEnglish language.Tag = language.MustParse("en-AU")
+ 183 | var languageMatcher = language.NewMatcher([]language.Tag{
+ 184 | language.English, // The first language is used as fallback.
+ 185 | AustralianEnglish,
+ 186 | language.BritishEnglish,
+ 187 | language.AmericanEnglish,
+ 188 | language.CanadianFrench,
+ 189 | language.French,
+ 190 | language.German,
+ 191 | language.Dutch,
+ 192 | Esperanto,
+ 193 | language.LatinAmericanSpanish,
+ 194 | language.EuropeanSpanish,
+ 195 | language.Spanish,
+ 196 | language.Danish,
+ 197 | language.TraditionalChinese,
+ 198 | language.SimplifiedChinese,
+ 199 | language.Chinese,
+ 200 | language.ModernStandardArabic,
+ 201 | language.Arabic,
+ 202 | language.Finnish,
+ 203 | language.Ukrainian,
+ 204 | language.Hebrew,
+ 205 | language.Italian,
+ 206 | language.BrazilianPortuguese,
+ 207 | language.EuropeanPortuguese,
+ 208 | language.Portuguese,
+ 209 | language.Russian,
+ 210 | language.Greek,
+ 211 | language.Hindi,
+ 212 | language.Korean,
+ 213 | language.Persian,
+ 214 | Yiddish, // Yiddish
+ 215 | language.Italian,
+ 216 | })
+ 217 |
+ 218 | func langTagToText(tag language.Tag) string {
+ 219 | switch tag {
+ 220 | case language.English:
+ 221 | return "English"
+ 222 | case language.BritishEnglish:
+ 223 | return "English"
+ 224 | case language.AmericanEnglish:
+ 225 | return "English"
+ 226 | case AustralianEnglish:
+ 227 | return "English"
+ 228 | case language.CanadianFrench:
+ 229 | return "French"
+ 230 | case language.French:
+ 231 | return "French"
+ 232 | case language.German:
+ 233 | return "German"
+ 234 | case language.Dutch:
+ 235 | return "Dutch"
+ 236 | case Esperanto:
+ 237 | return "Esperanto"
+ 238 | case language.LatinAmericanSpanish:
+ 239 | return "Spanish"
+ 240 | case language.EuropeanSpanish:
+ 241 | return "Spanish"
+ 242 | case language.Spanish:
+ 243 | return "Spanish"
+ 244 | case language.Danish:
+ 245 | return "Danish"
+ 246 | case language.TraditionalChinese:
+ 247 | return "Chinese"
+ 248 | case language.SimplifiedChinese:
+ 249 | return "Chinese"
+ 250 | case language.Chinese:
+ 251 | return "Chinese"
+ 252 | case language.ModernStandardArabic:
+ 253 | return "Arabic"
+ 254 | case language.Arabic:
+ 255 | return "Arabic"
+ 256 | case language.Finnish:
+ 257 | return "Finnish"
+ 258 | case language.Ukrainian:
+ 259 | return "Ukrainian"
+ 260 | case language.Hebrew:
+ 261 | return "Hebrew"
+ 262 | case language.Italian:
+ 263 | return "Italian"
+ 264 | case language.BrazilianPortuguese:
+ 265 | return "Portuguese"
+ 266 | case language.EuropeanPortuguese:
+ 267 | return "Portuguese"
+ 268 | case language.Portuguese:
+ 269 | return "Portuguese"
+ 270 | case language.Russian:
+ 271 | return "Russian"
+ 272 | case language.Greek:
+ 273 | return "Greek"
+ 274 | case language.Hindi:
+ 275 | return "Hindi"
+ 276 | case language.Korean:
+ 277 | return "Korean"
+ 278 | case language.Persian:
+ 279 | return "Persian"
+ 280 | case Yiddish:
+ 281 | return "Yiddish"
+ 282 | case language.Italian:
+ 283 | return "Italian"
+ 284 | }
+ 285 |
+ 286 | return ""
+ 287 | }
gemini/gemini.go
... | ...
2 | package gemini
3 |
4 | import (
5 | // "io"
6 |
7 | "fmt"
+ 7 | "net/http"
7 | "net/url"
8 | "strings"
9 | "unicode"
10 | "unicode/utf8"
11 |
... | ...
40 | if err != nil {
41 | panic(err)
42 | }
43 | context.GetPortListener("0.0.0.0", "1995").AddCertificate("auragem.letz.dev", "auragem.pem")
44 |
+ 46 | go startWebServer()
+ 47 |
45 | setupAuraGem(context)
46 | setupScholasticDiversity(context)
47 | setupScrollProtocol(context)
48 |
49 | context.Start()
... | ...
45 | setupAuraGem(context)
46 | setupScholasticDiversity(context)
47 | setupScrollProtocol(context)
48 |
49 | context.Start()
+ 53 | }
+ 54 |
+ 55 | func startWebServer() {
+ 56 | http.ListenAndServe("0.0.0.0:80", http.FileServer(http.Dir("/home/clseibold/ServerData/auragem_sis/SIS/scrollprotocol_http")))
50 | }
51 |
52 | func setupAuraGem(context *sis.SISContext) {
53 | geminiServer := context.AddServer(sis.Server{Type: sis.ServerType_Gemini, Name: "auragem_gemini", Hostname: "auragem.letz.dev", DefaultLanguage: "en"})
54 | context.GetPortListener("0.0.0.0", "1965").AddCertificate("auragem.letz.dev", "auragem.pem")
gemini/github.go
... | ...
25 | )
26 | tc := oauth2.NewClient(ctx, ts)
27 | client := github.NewClient(tc)
28 |
29 | g.AddRoute("/github", func(request sis.Request) {
+ 30 | request.SetClassification(sis.ScrollResponseUDC_Reference)
30 | request.Gemini(`# AuraGem Github Proxy
31 |
32 | Welcome to the AuraGem Github proxy!
33 |
34 | => /github/search Search Repos
... | ...
61 | handleGithubSearch(ctx, request, client, query, "")
62 | }
63 | })
64 |
65 | g.AddRoute("/github/repo/:id", func(request sis.Request) {
+ 67 | request.SetClassification(sis.ScrollResponseUDC_Docs)
66 | id := request.GetParam("id")
67 | template := `# Repo: %s
68 |
69 | %s
70 |
... | ...
110 | rootContents, _ := getRepoContents(ctx, client, repository, "")
111 | request.Gemini(fmt.Sprintf(template, repository.GetFullName(), repository.GetDescription(), repository.GetSSHURL(), repository.GetHTMLURL(), repository.GetHomepage(), repository.GetLicense().GetURL(), repository.GetLicense().GetName(), repository.GetID(), repository.GetID(), repository.GetDefaultBranch(), rootContents))
112 | })
113 |
114 | g.AddRoute("/github/repo/:id/b", func(request sis.Request) {
+ 117 | request.SetClassification(sis.ScrollResponseUDC_Docs)
115 | id := request.GetParam("id")
116 |
117 | id_int, err1 := strconv.Atoi(id)
118 | if err1 != nil {
119 | panic(err1)
... | ...
143 |
144 | request.Gemini(fmt.Sprintf(template, repository.GetFullName(), repository.GetID(), builder.String()))
145 | })
146 |
147 | g.AddRoute("/github/repo/:id/issues/", func(request sis.Request) {
+ 151 | request.SetClassification(sis.ScrollResponseUDC_Docs)
148 | id := request.GetParam("id")
149 |
150 | id_int, err1 := strconv.Atoi(id)
151 | if err1 != nil {
152 | panic(err1)
... | ...
180 | %s
181 | `, repository.GetFullName(), len(issues), repository.GetID(), builder.String()))
182 | })
183 |
184 | g.AddRoute("/github/repo/:id/issues/:issue", func(request sis.Request) {
+ 189 | request.SetClassification(sis.ScrollResponseUDC_Docs)
185 | id := request.GetParam("id")
186 | issueParam := request.GetParam("issue")
187 |
188 | id_int, err1 := strconv.Atoi(id)
189 | if err1 != nil {
gemini/music/music.go
... | ...
67 | //defer throttlePool.ReleasePool()
68 |
69 | handleRadioService(s, conn)
70 |
71 | s.AddRoute("/music/", func(request sis.Request) {
- 72 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# AuraGem Music\nA music service where you can upload a limited number of mp3s over Titan and listen to your private music library over Scroll/Gemini/Spartan. Stream individual songs or full albums, or use the \"Shuffled Stream\" feature that acts like a private radio of random songs from your library.\n"})
+ 72 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# AuraGem Music\nA music service where you can upload a limited number of mp3s over Titan and listen to your private music library over Scroll/Gemini/Spartan. Stream individual songs or full albums, or use the \"Shuffled Stream\" feature that acts like a private radio of random songs from your library.\n"})
73 | if request.ScrollMetadataRequested {
74 | request.SendAbstract("")
75 | return
76 | }
77 |
... | ...
105 | }
106 | }
107 | })
108 |
109 | s.AddRoute("/music/quota", func(request sis.Request) {
- 110 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# AuraGem Music - How the Quota System Works\nDescribes the quota system.\n"})
+ 110 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Docs, PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# AuraGem Music - How the Quota System Works\nDescribes the quota system.\n"})
111 | if request.ScrollMetadataRequested {
112 | request.SendAbstract("")
113 | return
114 | }
115 | template := `# AuraGem Music - How the Quota System Works
... | ...
126 | `
127 | request.Gemini(fmt.Sprintf(template, userSongQuota))
128 | })
129 |
130 | s.AddRoute("/music/about", func(request sis.Request) {
- 131 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# About AuraGem Music\n"})
+ 131 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Docs, PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# About AuraGem Music\n"})
132 | if request.ScrollMetadataRequested {
133 | request.SendAbstract("")
134 | return
135 | }
136 | template := `# About AuraGem Music
... | ...
162 | }
163 | openFile, err := os.Open(filepath.Join(musicDirectory, file.Filename))
164 | if err != nil {
165 | panic(err)
166 | }
- 167 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# Random Music File\n"})
+ 167 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# Random Music File\n", Classification: sis.ScrollResponseUDC_Music})
168 | request.SetNoLanguage()
169 | if request.ScrollMetadataRequested {
170 | request.SendAbstract("audio/mpeg")
171 | return
172 | }
... | ...
193 | }
194 | uploadLink = "=> " + titanHost + "/music/upload"
195 | uploadMethod = "Titan"
196 | }
197 |
- 198 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# Upload File with " + uploadMethod + "\n"})
+ 198 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Docs, PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# Upload File with " + uploadMethod + "\n"})
199 | if request.ScrollMetadataRequested {
200 | request.SendAbstract("")
201 | return
202 | }
203 |
... | ...
376 | }
377 | abstract += "Kbps: " + strconv.Itoa(int(file.CbrKbps)) + "\n"
378 | if file.Attribution != "" {
379 | abstract += "\nAttribution:\n" + file.Attribution + "\n"
380 | }
- 381 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: file.Artist, Abstract: abstract})
+ 381 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: file.Artist, Abstract: abstract, Classification: sis.ScrollResponseUDC_Music})
382 | request.SetNoLanguage()
383 | if request.ScrollMetadataRequested {
384 | request.SendAbstract("audio/mpeg")
385 | return
386 | }
... | ...
403 | request.Gemini(registerNotification)
404 | return
405 | } else {
406 | albums := GetAlbumsInUserLibrary(conn, user.Id)
407 |
- 408 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: user.Date_joined, Abstract: "# AuraGem Music - " + user.Username + "\n## Albums\n"})
+ 408 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, PublishDate: user.Date_joined, Abstract: "# AuraGem Music - " + user.Username + "\n## Albums\n"})
409 | if request.ScrollMetadataRequested {
410 | request.SendAbstract("")
411 | return
412 | }
413 |
... | ...
439 | request.Gemini(registerNotification)
440 | return
441 | } else {
442 | artists := GetArtistsInUserLibrary(conn, user.Id)
443 |
- 444 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: user.Date_joined, Abstract: "# AuraGem Music - " + user.Username + "\n## Artists\n"})
+ 444 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, PublishDate: user.Date_joined, Abstract: "# AuraGem Music - " + user.Username + "\n## Artists\n"})
445 | if request.ScrollMetadataRequested {
446 | request.SendAbstract("")
447 | return
448 | }
449 |
... | ...
531 | user, isRegistered := GetUser(conn, request.UserCertHash_Gemini())
532 | if !isRegistered {
533 | request.Gemini(registerNotification)
534 | return
535 | } else {
- 536 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# AuraGem Music Shuffled Stream - " + user.Username + "\n"})
+ 536 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Abstract: "# AuraGem Music Shuffled Stream - " + user.Username + "\n"})
537 | if request.ScrollMetadataRequested {
538 | request.SendAbstract("audio/mpeg")
539 | return
540 | }
541 |
... | ...
693 | err := row.Scan(&numRows)
694 | if err != nil {
695 | panic(err)
696 | }
697 | if numRows < 1 {
- 698 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# AuraGem Music - Register User " + username + "\n"})
+ 698 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Unclassed, Abstract: "# AuraGem Music - Register User " + username + "\n"})
699 | if request.ScrollMetadataRequested {
700 | request.SendAbstract("")
701 | return
702 | }
703 |
... | ...
761 |
762 | request.Gemini(fmt.Sprintf(template, user.Username, user.QuotaCount, userSongQuota, user.QuotaCount/float64(userSongQuota)*100, builder.String()))
763 | }
764 |
765 | func artistAlbums(request sis.Request, conn *sql.DB, user MusicUser, artist string) {
- 766 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# AuraGem Music - " + user.Username + "\n## Artist Albums: " + artist + "\n"})
+ 766 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Abstract: "# AuraGem Music - " + user.Username + "\n## Artist Albums: " + artist + "\n"})
767 | if request.ScrollMetadataRequested {
768 | request.SendAbstract("")
769 | return
770 | }
771 |
... | ...
786 | %s
787 | `, user.Username, artist, url.PathEscape(artist), builder.String()))
788 | }
789 |
790 | func albumSongs(request sis.Request, conn *sql.DB, user MusicUser, artist string, album string) {
- 791 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# AuraGem Music - " + user.Username + "\n## Album: " + album + " by " + artist + "\n"})
+ 791 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Abstract: "# AuraGem Music - " + user.Username + "\n## Album: " + album + " by " + artist + "\n"})
792 | if request.ScrollMetadataRequested {
793 | request.SendAbstract("")
794 | return
795 | }
796 |
... | ...
819 | %s
820 | `, user.Username, album, artist, url.PathEscape(albumartist), albumartist, url.PathEscape(albumartist), url.PathEscape(album), builder.String()))
821 | }
822 |
823 | func adminPage(request sis.Request, conn *sql.DB, user MusicUser) {
- 824 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# AuraGem Music - Admin\n"})
+ 824 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Unclassed, Abstract: "# AuraGem Music - Admin\n"})
825 | if request.ScrollMetadataRequested {
826 | request.SendAbstract("")
827 | return
828 | }
829 |
... | ...
856 |
857 | request.Gemini(fmt.Sprintf(template, globalQuotaCount, globalSongQuota, globalQuotaCount/globalSongQuota*100, avgUserQuotaCount, userSongQuota, avgUserQuotaCount/float64(userSongQuota)*100, userCount, artistCount, albumCount, builder.String()))
858 | }
859 |
860 | func adminGenrePage(request sis.Request, conn *sql.DB, user MusicUser, genre_string string) {
- 861 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# AuraGem Music - Admin: Genre" + genre_string + "\n"})
+ 861 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Unclassed, Abstract: "# AuraGem Music - Admin: Genre" + genre_string + "\n"})
862 | if request.ScrollMetadataRequested {
863 | request.SendAbstract("")
864 | return
865 | }
866 |
... | ...
878 | `, genre_string, builder.String()))
879 | }
880 |
881 | // Streams all songs in album in one streams
882 | func streamAlbumSongs(request sis.Request, conn *sql.DB, user MusicUser, artist string, album string) {
- 883 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# Stream Album " + album + " by " + artist + "\n"})
+ 883 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Abstract: "# Stream Album " + album + " by " + artist + "\n"})
884 | if request.ScrollMetadataRequested {
885 | request.SendAbstract("")
886 | return
887 | }
888 |
... | ...
896 |
897 | StreamMultipleFiles(request, musicFiles)
898 | }
899 |
900 | func streamArtistSongs(request sis.Request, conn *sql.DB, user MusicUser, artist string) {
- 901 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# Stream Songs by " + artist + "\n"})
+ 901 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Abstract: "# Stream Songs by " + artist + "\n"})
902 | if request.ScrollMetadataRequested {
903 | request.SendAbstract("")
904 | return
905 | }
906 |
... | ...
915 | }
916 |
917 | // ----- Manage Library Functions -----
918 |
919 | func manageLibrary(request sis.Request, user MusicUser) {
- 920 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: user.Date_joined, Abstract: "# Manage Library - " + user.Username + "\n"})
+ 920 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Unclassed, PublishDate: user.Date_joined, Abstract: "# Manage Library - " + user.Username + "\n"})
921 | if request.ScrollMetadataRequested {
922 | request.SendAbstract("")
923 | return
924 | }
925 |
... | ...
934 |
935 | `, user.Username))
936 | }
937 |
938 | func manageLibrary_deleteSelection(request sis.Request, conn *sql.DB, user MusicUser) {
- 939 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# Manage Library: Delete Selection - " + user.Username + "\n"})
+ 939 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Unclassed, Abstract: "# Manage Library: Delete Selection - " + user.Username + "\n"})
940 | if request.ScrollMetadataRequested {
941 | request.SendAbstract("")
942 | return
943 | }
944 |
... | ...
975 | if !exists {
976 | request.TemporaryFailure("File not in user library.")
977 | return
978 | }
979 |
- 980 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Abstract: "# Manage Library: Delete File - " + user.Username + "\nDelete " + file.Title + " by " + file.Artist + " (" + file.Album + ")\n"})
+ 980 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Unclassed, Abstract: "# Manage Library: Delete File - " + user.Username + "\nDelete " + file.Title + " by " + file.Artist + " (" + file.Album + ")\n"})
981 | if request.ScrollMetadataRequested {
982 | request.SendAbstract("")
983 | return
984 | }
985 |
gemini/music/radio.go
... | ...
10 | "path/filepath"
11 | "strings"
12 | "sync"
13 | "time"
14 |
+ 15 | "github.com/dhowden/tag"
15 | "github.com/gammazero/deque"
16 | sis "gitlab.com/clseibold/smallnetinformationservices"
17 | )
18 |
19 | type RadioBufInterface interface {
... | ...
87 | return MusicFile{}, true, "", err // TODO: This locks forever
88 | }
89 | //rb.File = f
90 |
91 | // Skip ID3v2 Tags at start of file
- 92 | skip_err := SkipId3HeaderTags(f)
+ 93 | skip_err := tag.SkipID3v2Tags(f)
93 | if skip_err != nil {
94 | fmt.Printf("Failed to skip ID3 Headers\n")
95 | }
96 |
97 | // Set starting location to after tags, set the bitrate, update the fileChangeIndex, unlock the lock, and broadcast that the new song was selected
... | ...
170 | updateDate, _ := time.ParseInLocation(time.RFC3339, "2023-11-30T00:00:00", time.Local)
171 | abstract := `# AuraGem Music: Public Radio
172 |
173 | This is AuraGem Music's public radio that plays public domain and royalty free music. All music is collected from sources like the Free Music Archive, archive.org, and Chosic, and stored on my server. This radio does not proxy from the web, unlike other radios over on Gopherspace.
174 | `
- 175 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: creationDate.UTC(), UpdateDate: updateDate.UTC(), Language: "en", Abstract: abstract})
+ 176 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Author: "Christian Lee Seibold", PublishDate: creationDate.UTC(), UpdateDate: updateDate.UTC(), Language: "en", Abstract: abstract})
176 | if request.ScrollMetadataRequested {
177 | request.SendAbstract("")
178 | return
179 | }
180 |
gemini/music/stations.go
... | ...
30 |
31 | s.AddRoute("/music/public_radio/"+url.PathEscape(station.Name), func(request sis.Request) {
32 | creationDate, _ := time.ParseInLocation(time.RFC3339, "2024-03-14T18:07:00", time.Local)
33 | creationDate = creationDate.UTC()
34 | abstract := fmt.Sprintf("# AuraGem Public Radio - %s Station\n\n%s\nClients Connected: %d\n", station.Name, station.Description, radioBuffer.clientCount)
- 35 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: creationDate, UpdateDate: creationDate, Language: "en", Abstract: abstract})
+ 35 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Author: "Christian Lee Seibold", PublishDate: creationDate, UpdateDate: creationDate, Language: "en", Abstract: abstract})
36 | if request.ScrollMetadataRequested {
37 | request.SendAbstract("")
38 | return
39 | }
40 |
... | ...
165 |
166 | s.AddRoute("/music/public_radio/"+url.PathEscape(station.Name)+"/schedule_feed", func(request sis.Request) {
167 | creationDate, _ := time.ParseInLocation(time.RFC3339, "2024-03-14T18:07:00", time.Local)
168 | creationDate = creationDate.UTC()
169 | abstract := fmt.Sprintf("# AuraGem Public Radio - %s Station Schedule\n", station.Name)
- 170 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: creationDate, UpdateDate: time.Now(), Language: "en", Abstract: abstract})
+ 170 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Author: "Christian Lee Seibold", PublishDate: creationDate, UpdateDate: time.Now(), Language: "en", Abstract: abstract})
171 | if request.ScrollMetadataRequested {
172 | request.SendAbstract("")
173 | return
174 | }
175 |
... | ...
230 | }
231 |
232 | abstract = fmt.Sprintf("# AuraGem Public Radio - %s Station\n\n%s\nClients Currently Connected to Station: %d\nCurrent Time and Genre: %s CST (%s)\nCurrent song playing: %s by %s\n%s", station.Name, station.Description, radioBuffer.clientCount, currentTime.Format("03:04 PM"), radioGenre, radioBuffer.currentMusicFile.Title, radioBuffer.currentMusicFile.Artist, attribution)
233 | }
234 |
- 235 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: creationDate, UpdateDate: time.Now(), Language: "en", Abstract: abstract})
+ 235 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Music, Author: "Christian Lee Seibold", PublishDate: creationDate, UpdateDate: time.Now(), Language: "en", Abstract: abstract})
236 | if request.ScrollMetadataRequested {
237 | request.SendAbstract("audio/mpeg")
238 | return
239 | }
240 |
gemini/search/db.go
... | ...
12 | "unicode/utf8"
13 | // "strconv"
14 | )
15 |
16 | func getRecent(conn *sql.DB) []Page {
- 17 | q := `SELECT FIRST 50 id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE hidden=false ORDER BY date_added DESC`
+ 17 | q := `SELECT FIRST 50 id, url, scheme, domainid, contenttype, charset, language, linecount, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE hidden=false ORDER BY date_added DESC`
18 |
19 | rows, rows_err := conn.QueryContext(context.Background(), q)
20 |
21 | var pages []Page = make([]Page, 0, 50)
22 | if rows_err == nil {
... | ...
21 | var pages []Page = make([]Page, 0, 50)
22 | if rows_err == nil {
23 | defer rows.Close()
24 | for rows.Next() {
25 | var page Page
- 26 | scan_err := rows.Scan(&page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 26 | scan_err := rows.Scan(&page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
27 | if scan_err == nil {
28 | pages = append(pages, page)
29 | } else {
30 | prevPage := Page{}
31 | if len(pages) > 0 {
... | ...
100 | return tags
101 | }
102 |
103 | // TODO: Set a way to order the results
104 | func getPagesOfTag(conn *sql.DB, name string) []Page {
- 105 | q := `SELECT COUNT(*) OVER () as total, pages.id, pages.url, pages.scheme, pages.domainid, pages.contenttype, pages.charset, pages.language, pages.linecount, pages.title, pages.prompt, pages.size, pages.hash, pages.feed, pages.publishdate, pages.indextime, pages.album, pages.artist, pages.albumartist, pages.composer, pages.track, pages.disc, pages.copyright, pages.crawlindex, pages.date_added, pages.last_successful_visit, pages.hidden FROM tags JOIN pages ON pages.id = tags.pageid where tags.name=?`
+ 105 | q := `SELECT COUNT(*) OVER () as total, pages.id, pages.url, pages.scheme, pages.domainid, pages.contenttype, pages.charset, pages.language, pages.linecount, pages.udc, pages.title, pages.prompt, pages.size, pages.hash, pages.feed, pages.publishdate, pages.indextime, pages.album, pages.artist, pages.albumartist, pages.composer, pages.track, pages.disc, pages.copyright, pages.crawlindex, pages.date_added, pages.last_successful_visit, pages.hidden FROM tags JOIN pages ON pages.id = tags.pageid where tags.name=?`
106 |
107 | rows, rows_err := conn.QueryContext(context.Background(), q, name)
108 |
109 | var pages []Page = nil
110 | if rows_err == nil {
... | ...
110 | if rows_err == nil {
111 | var count int64
112 | defer rows.Close()
113 | for rows.Next() {
114 | var page Page
- 115 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 115 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
116 | if scan_err == nil {
117 | if pages == nil {
118 | pages = make([]Page, 0, count)
119 | }
120 | pages = append(pages, page)
... | ...
134 |
135 | return pages
136 | }
137 |
138 | func getMimetypeFiles(conn *sql.DB, mimetype string) []Page {
- 139 | q := `SELECT FIRST 300 id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE contenttype=? AND hidden=false`
+ 139 | q := `SELECT FIRST 300 id, url, scheme, domainid, contenttype, charset, language, linecount, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE contenttype=? AND hidden=false`
140 |
141 | rows, rows_err := conn.QueryContext(context.Background(), q, mimetype)
142 |
143 | var pages []Page = make([]Page, 0, 300)
144 | if rows_err == nil {
... | ...
143 | var pages []Page = make([]Page, 0, 300)
144 | if rows_err == nil {
145 | defer rows.Close()
146 | for rows.Next() {
147 | var page Page
- 148 | scan_err := rows.Scan(&page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 148 | scan_err := rows.Scan(&page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
149 | if scan_err == nil {
150 | pages = append(pages, page)
151 | } else {
152 | prevPage := Page{}
153 | if len(pages) > 0 {
... | ...
192 |
193 | return mimetypes
194 | }
195 |
196 | func getFeeds(conn *sql.DB) []Page {
- 197 | rows, rows_err := conn.QueryContext(context.Background(), "SELECT COUNT(*) OVER () as total, id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE feed = true AND hidden = false")
+ 197 | rows, rows_err := conn.QueryContext(context.Background(), "SELECT COUNT(*) OVER () as total, id, url, scheme, domainid, contenttype, charset, language, linecount, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE feed = true AND hidden = false")
198 |
199 | var pages []Page = nil
200 | if rows_err == nil {
201 | var count int64
202 | defer rows.Close()
... | ...
200 | if rows_err == nil {
201 | var count int64
202 | defer rows.Close()
203 | for rows.Next() {
204 | var page Page
- 205 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 205 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
206 | if scan_err == nil {
207 | if pages == nil {
208 | pages = make([]Page, 0, count)
209 | }
210 | pages = append(pages, page)
... | ...
224 |
225 | return pages
226 | }
227 |
228 | func getPagesWithPublishDate(conn *sql.DB) []Page {
- 229 | rows, rows_err := conn.QueryContext(context.Background(), "SELECT COUNT(*) OVER () as total, id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE publishdate <> ? AND hidden = false ORDER BY publishdate DESC", time.Time{})
+ 229 | rows, rows_err := conn.QueryContext(context.Background(), "SELECT COUNT(*) OVER () as total, id, url, scheme, domainid, contenttype, charset, language, linecount, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE publishdate <> ? AND hidden = false ORDER BY publishdate DESC", time.Time{})
230 |
231 | var pages []Page = nil
232 | if rows_err == nil {
233 | var count int64
234 | defer rows.Close()
... | ...
232 | if rows_err == nil {
233 | var count int64
234 | defer rows.Close()
235 | for rows.Next() {
236 | var page Page
- 237 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 237 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
238 | if scan_err == nil {
239 | if pages == nil {
240 | pages = make([]Page, 0, count)
241 | }
242 | pages = append(pages, page)
... | ...
258 | }
259 |
260 | // TODO: Allow for different languages
261 | // NOTE: Blank language fields are considered English
262 | func getPagesWithPublishDateFromLastYear(conn *sql.DB, results int, skip int) ([]Page, int) {
- 263 | query := fmt.Sprintf("SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE publishdate > dateadd(-1 year to ?) AND publishdate < dateadd(2 day to ?) AND (language = '' OR language LIKE 'en%%') AND hidden = false ORDER BY publishdate DESC", results, skip)
+ 263 | query := fmt.Sprintf("SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, linecount, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE publishdate > dateadd(-1 year to ?) AND publishdate < dateadd(2 day to ?) AND (language = '' OR language LIKE 'en%%') AND hidden = false ORDER BY publishdate DESC", results, skip)
264 | rows, rows_err := conn.QueryContext(context.Background(), query, time.Now().UTC(), time.Now().UTC())
265 |
266 | var pages []Page = make([]Page, 0, results)
267 | var totalCount int
268 | if rows_err == nil {
... | ...
267 | var totalCount int
268 | if rows_err == nil {
269 | defer rows.Close()
270 | for rows.Next() {
271 | var page Page
- 272 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 272 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
273 | if scan_err == nil {
274 | pages = append(pages, page)
275 | } else {
276 | prevPage := Page{}
277 | if len(pages) > 0 {
... | ...
291 |
292 | // Returns []Page, totalResultsCount, and whether there's a next page
293 | func getAudioFiles(conn *sql.DB, page int64) ([]Page, int64, bool) {
294 | var results int64 = 30
295 | skip := (page - 1) * results
- 296 | q := fmt.Sprintf(`SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE contenttype IN ('audio/mpeg', 'audio/mp3', 'audio/ogg', 'audio/flac', 'audio/mid', 'audio/m4a', 'audio/x-flac') AND hidden = false`, results, skip)
+ 296 | q := fmt.Sprintf(`SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE contenttype IN ('audio/mpeg', 'audio/mp3', 'audio/ogg', 'audio/flac', 'audio/mid', 'audio/m4a', 'audio/x-flac') AND hidden = false`, results, skip)
297 |
298 | rows, rows_err := conn.QueryContext(context.Background(), q)
299 |
300 | var pages []Page = make([]Page, 0, results)
301 | var totalCount int64
... | ...
301 | var totalCount int64
302 | if rows_err == nil {
303 | defer rows.Close()
304 | for rows.Next() {
305 | var page Page
- 306 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 306 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
307 | if scan_err == nil {
308 | pages = append(pages, page)
309 | } else {
310 | prevPage := Page{}
311 | if len(pages) > 0 {
... | ...
325 |
326 | // Returns []Page, totalResultsCount, and whether there's a next page
327 | func getImageFiles(conn *sql.DB, page int64) ([]Page, int64, bool) {
328 | var results int64 = 30
329 | skip := (page - 1) * results
- 330 | q := fmt.Sprintf(`SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE contenttype IN ('image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/bmp', 'image/webp', 'image/svg+xml', 'image/vnd.mozilla.apng') AND hidden = false`, results, skip)
+ 330 | q := fmt.Sprintf(`SELECT FIRST %d SKIP %d COUNT(*) OVER () totalCount, id, url, scheme, domainid, contenttype, charset, language, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE contenttype IN ('image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/bmp', 'image/webp', 'image/svg+xml', 'image/vnd.mozilla.apng') AND hidden = false`, results, skip)
331 |
332 | rows, rows_err := conn.QueryContext(context.Background(), q)
333 |
334 | var pages []Page = make([]Page, 0, results)
335 | var totalCount int64
... | ...
335 | var totalCount int64
336 | if rows_err == nil {
337 | defer rows.Close()
338 | for rows.Next() {
339 | var page Page
- 340 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 340 | scan_err := rows.Scan(&totalCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
341 | if scan_err == nil {
342 | pages = append(pages, page)
343 | } else {
344 | prevPage := Page{}
345 | if len(pages) > 0 {
... | ...
356 |
357 | return pages, totalCount, skip+results < totalCount
358 | }
359 |
360 | func getTwtxtFiles(conn *sql.DB) []Page {
- 361 | q := `SELECT COUNT(*) OVER () as total, id, url, scheme, domainid, contenttype, charset, language, linecount, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE (url LIKE '%twtxt.txt' OR url LIKE '%tw.txt') AND hidden = false`
+ 361 | q := `SELECT COUNT(*) OVER () as total, id, url, scheme, domainid, contenttype, charset, language, linecount, udc, title, prompt, size, hash, feed, publishdate, indextime, album, artist, albumartist, composer, track, disc, copyright, crawlindex, date_added, last_successful_visit, hidden FROM pages WHERE (url LIKE '%twtxt.txt' OR url LIKE '%tw.txt') AND hidden = false`
362 |
363 | rows, rows_err := conn.QueryContext(context.Background(), q)
364 |
365 | var pages []Page = nil
366 | if rows_err == nil {
... | ...
366 | if rows_err == nil {
367 | var count int64
368 | defer rows.Close()
369 | for rows.Next() {
370 | var page Page
- 371 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 371 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
372 | if scan_err == nil {
373 | if pages == nil {
374 | pages = make([]Page, 0, count)
375 | }
376 | pages = append(pages, page)
... | ...
390 |
391 | return pages
392 | }
393 |
394 | func getSecurityTxtFiles(conn *sql.DB) []PageWithDomain {
- 395 | q := `SELECT COUNT(*) OVER () as total, pages.id, pages.url, pages.scheme, pages.domainid, pages.contenttype, pages.charset, pages.language, pages.linecount, pages.title, pages.prompt, pages.size, pages.hash, pages.feed, pages.publishdate, pages.indextime, pages.album, pages.artist, pages.albumartist, pages.composer, pages.track, pages.disc, pages.copyright, pages.crawlindex, pages.date_added, pages.last_successful_visit, pages.hidden, domains.id, domains.domain, domains.title, domains.port, domains.has_robots, domains.has_security, domains.has_favicon, domains.favicon, domains.crawlindex, domains.date_added FROM pages INNER JOIN domains ON domains.ID = pages.domainid WHERE pages.url LIKE '%security.txt' AND hidden = false`
+ 395 | q := `SELECT COUNT(*) OVER () as total, pages.id, pages.url, pages.scheme, pages.domainid, pages.contenttype, pages.charset, pages.language, pages.linecount, pages.udc, pages.title, pages.prompt, pages.size, pages.hash, pages.feed, pages.publishdate, pages.indextime, pages.album, pages.artist, pages.albumartist, pages.composer, pages.track, pages.disc, pages.copyright, pages.crawlindex, pages.date_added, pages.last_successful_visit, pages.hidden, domains.id, domains.domain, domains.title, domains.port, domains.has_robots, domains.has_security, domains.has_favicon, domains.favicon, domains.crawlindex, domains.date_added FROM pages INNER JOIN domains ON domains.ID = pages.domainid WHERE pages.url LIKE '%security.txt' AND hidden = false`
396 |
397 | rows, rows_err := conn.QueryContext(context.Background(), q)
398 |
399 | var pages []PageWithDomain = nil
400 | if rows_err == nil {
... | ...
401 | var count int64
402 | defer rows.Close()
403 | for rows.Next() {
404 | var page Page
405 | var domain Domain
- 406 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden, &domain.Id, &domain.Domain, &domain.Title, &domain.Port, &domain.HasRobots, &domain.HasSecurity, &domain.HasFavicon, &domain.Favicon, &domain.CrawlIndex, &domain.Date_added)
+ 406 | scan_err := rows.Scan(&count, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden, &domain.Id, &domain.Domain, &domain.Title, &domain.Port, &domain.HasRobots, &domain.HasSecurity, &domain.HasFavicon, &domain.Favicon, &domain.CrawlIndex, &domain.Date_added)
407 | if scan_err == nil {
408 | if pages == nil {
409 | pages = make([]PageWithDomain, 0, count)
410 | }
411 | pages = append(pages, PageWithDomain{page, domain})
gemini/search/search.go
... | ...
38 | GROUP BY ID, URL, SCHEME, DOMAINID, CONTENTTYPE, CHARSET, LANGUAGE, LINECOUNT, TITLE, PROMPT, SIZE, HASH, FEED, PUBLISHDATE, INDEXTIME, ALBUM, ARTIST, ALBUMARTIST, COMPOSER, TRACK, DISC, COPYRIGHT, CRAWLINDEX, DATE_ADDED, LAST_SUCCESSFUL_VISIT, HIDDEN
39 | ORDER BY GROUPED_SCORE DESC, s.publishdate DESC`*/
40 |
41 | // FTS.FTS$ID as fts_id
42 | var fts_searchQuery string = `
- 43 | select FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, (FTS.FTS$SCORE) as GROUPED_SCORE, P.ID, P.URL, P.SCHEME, P.DOMAINID, P.CONTENTTYPE, P.CHARSET, P.LANGUAGE, P.LINECOUNT, P.TITLE, P.PROMPT, P.SIZE, P.HASH, P.FEED, CASE WHEN EXTRACT(YEAR FROM P.PUBLISHDATE) < 1800 THEN TIMESTAMP '01.01.9999 00:00:00.000' ELSE P.PUBLISHDATE END AS PUBLISHDATE, P.INDEXTIME, P.ALBUM, P.ARTIST, P.ALBUMARTIST, P.COMPOSER, P.TRACK, P.DISC, P.COPYRIGHT, P.CRAWLINDEX, P.DATE_ADDED, P.LAST_SUCCESSFUL_VISIT, P.HIDDEN
+ 43 | select FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, (FTS.FTS$SCORE) as GROUPED_SCORE, P.ID, P.URL, P.SCHEME, P.DOMAINID, P.CONTENTTYPE, P.CHARSET, P.LANGUAGE, P.LINECOUNT, P.UDC, P.TITLE, P.PROMPT, P.SIZE, P.HASH, P.FEED, CASE WHEN EXTRACT(YEAR FROM P.PUBLISHDATE) < 1800 THEN TIMESTAMP '01.01.9999 00:00:00.000' ELSE P.PUBLISHDATE END AS PUBLISHDATE, P.INDEXTIME, P.ALBUM, P.ARTIST, P.ALBUMARTIST, P.COMPOSER, P.TRACK, P.DISC, P.COPYRIGHT, P.CRAWLINDEX, P.DATE_ADDED, P.LAST_SUCCESSFUL_VISIT, P.HIDDEN
44 | FROM FTS$SEARCH('FTS_PAGE_ID_EN', '(%%query%%) AND HIDDEN:false') FTS
45 | JOIN PAGES P ON P.ID = FTS.FTS$ID
46 | ORDER BY GROUPED_SCORE DESC, PUBLISHDATE DESC, CHAR_LENGTH(P.URL) ASC
47 | `
48 | var fts_searchQuery_protocol string = `
... | ...
44 | FROM FTS$SEARCH('FTS_PAGE_ID_EN', '(%%query%%) AND HIDDEN:false') FTS
45 | JOIN PAGES P ON P.ID = FTS.FTS$ID
46 | ORDER BY GROUPED_SCORE DESC, PUBLISHDATE DESC, CHAR_LENGTH(P.URL) ASC
47 | `
48 | var fts_searchQuery_protocol string = `
- 49 | select FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, (FTS.FTS$SCORE) as GROUPED_SCORE, P.ID, P.URL, P.SCHEME, P.DOMAINID, P.CONTENTTYPE, P.CHARSET, P.LANGUAGE, P.LINECOUNT, P.TITLE, P.PROMPT, P.SIZE, P.HASH, P.FEED, CASE WHEN EXTRACT(YEAR FROM P.PUBLISHDATE) < 1800 THEN TIMESTAMP '01.01.9999 00:00:00.000' ELSE P.PUBLISHDATE END AS PUBLISHDATE, P.INDEXTIME, P.ALBUM, P.ARTIST, P.ALBUMARTIST, P.COMPOSER, P.TRACK, P.DISC, P.COPYRIGHT, P.CRAWLINDEX, P.DATE_ADDED, P.LAST_SUCCESSFUL_VISIT, P.HIDDEN
+ 49 | select FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, (FTS.FTS$SCORE) as GROUPED_SCORE, P.ID, P.URL, P.SCHEME, P.DOMAINID, P.CONTENTTYPE, P.CHARSET, P.LANGUAGE, P.LINECOUNT, P.UDC, P.TITLE, P.PROMPT, P.SIZE, P.HASH, P.FEED, CASE WHEN EXTRACT(YEAR FROM P.PUBLISHDATE) < 1800 THEN TIMESTAMP '01.01.9999 00:00:00.000' ELSE P.PUBLISHDATE END AS PUBLISHDATE, P.INDEXTIME, P.ALBUM, P.ARTIST, P.ALBUMARTIST, P.COMPOSER, P.TRACK, P.DISC, P.COPYRIGHT, P.CRAWLINDEX, P.DATE_ADDED, P.LAST_SUCCESSFUL_VISIT, P.HIDDEN
50 | FROM FTS$SEARCH('FTS_PAGE_ID_EN', '(%%query%%) AND HIDDEN:false AND SCHEME:%%protocol%%') FTS
51 | JOIN PAGES P ON P.ID = FTS.FTS$ID
52 | ORDER BY GROUPED_SCORE DESC, PUBLISHDATE DESC, CHAR_LENGTH(P.URL) ASC
53 | `
54 |
... | ...
51 | JOIN PAGES P ON P.ID = FTS.FTS$ID
52 | ORDER BY GROUPED_SCORE DESC, PUBLISHDATE DESC, CHAR_LENGTH(P.URL) ASC
53 | `
54 |
55 | var fts_audioSearchQuery string = `
- 56 | select FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, SUM(s.SCORE) as GROUPED_SCORE, s.HIGHLIGHT, s.ID, s.URL, s.SCHEME, s.DOMAINID, s.CONTENTTYPE, s.CHARSET, s.LANGUAGE, s.LINECOUNT, s.TITLE, s.PROMPT, s.SIZE, s.HASH, s.FEED, s.PUBLISHDATE, s.INDEXTIME, s.ALBUM, s.ARTIST, s.ALBUMARTIST, s.COMPOSER, s.TRACK, s.DISC, s.COPYRIGHT, s.CRAWLINDEX, s.DATE_ADDED, s.LAST_SUCCESSFUL_VISIT, s.HIDDEN
+ 56 | select FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, SUM(s.SCORE) as GROUPED_SCORE, s.HIGHLIGHT, s.ID, s.URL, s.SCHEME, s.DOMAINID, s.CONTENTTYPE, s.CHARSET, s.LANGUAGE, s.LINECOUNT, s.UDC, s.TITLE, s.PROMPT, s.SIZE, s.HASH, s.FEED, s.PUBLISHDATE, s.INDEXTIME, s.ALBUM, s.ARTIST, s.ALBUMARTIST, s.COMPOSER, s.TRACK, s.DISC, s.COPYRIGHT, s.CRAWLINDEX, s.DATE_ADDED, s.LAST_SUCCESSFUL_VISIT, s.HIDDEN
57 | FROM (select FTS.FTS$ID as fts_id, FTS.FTS$SCORE as SCORE,
58 | FTS$HIGHLIGHTER.FTS$BEST_FRAGMENT(A.TEXT, '%%query%%', 'ENGLISH', 'TEXT', 70, '[', ']') AS HIGHLIGHT,
59 | P.*
60 | FROM FTS$SEARCH('FTS_AUDIOTRANSCRIPT_ID_EN', '%%query%%') FTS
61 | JOIN AUDIOTRANSCRIPTS A ON A.ID = FTS.FTS$ID
... | ...
145 | request.SendAbstract("")
146 | return
147 | }
148 |
149 | request.Gemini("# AuraGem Search\n\n")
- 150 | request.PromptLine("/search/s/", "🔍 Search")
+ 150 | request.PromptLine("/search/s/", "🔍 Search Smallnet")
+ 151 | request.PromptLine("/search/gemini/", "🔍 Search Geminispace")
+ 152 | request.PromptLine("/search/scroll/", "🔍 Search Scrollspace")
151 | request.Gemini(`=> /search/random/ 🎲 Goto Random Capsule
152 | => /search/backlinks/ Check Backlinks
153 |
154 | => /search/features/ About and Features
155 | => /search/stats/ 📈 Statistics
... | ...
198 | //
199 | // => https://www.patreon.com/krixano Patreon
200 | })
201 |
202 | s.AddRoute("/search/configure_default", func(request sis.Request) {
- 203 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# Configure Default Search Engine in Lagrange\n"})
+ 205 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Docs, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# Configure Default Search Engine in Lagrange\n"})
204 | if request.ScrollMetadataRequested {
205 | request.SendAbstract("")
206 | return
207 | }
208 |
... | ...
213 | > gemini://auragem.letz.dev/search/s
214 | `)
215 | })
216 |
217 | s.AddRoute("/search/features", func(request sis.Request) {
- 218 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search Features\n"})
+ 220 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Docs, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search Features\n"})
219 | if request.ScrollMetadataRequested {
220 | request.SendAbstract("")
221 | return
222 | }
223 |
... | ...
304 | request.SendAbstract("")
305 | return
306 | }
307 |
308 | if totalSizeCache == -1 || lastCacheTime.Add(refreshCacheEvery).Before(currentTime) {
- 309 | row := conn.QueryRowContext(context.Background(), "SELECT COUNT(*), MAX(LAST_SUCCESSFUL_VISIT), SUM(SIZE) FROM pages")
+ 311 | row := conn.QueryRowContext(context.Background(), "SELECT COUNT(*), MAX(LAST_SUCCESSFUL_VISIT), SUM(SIZE) FROM pages WHERE SCHEME = 'gemini' OR SCHEME = 'GEMINI'")
310 | row.Scan(&pagesCountCache, &lastCrawlCache, &totalSizeCache)
311 | // Convert totalSize to GB
312 | lastCacheTime = currentTime
313 | }
314 |
... | ...
324 | row3 := conn.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM pages WHERE FEED = true")
325 | feedCount := 0
326 | row3.Scan(&feedCount)
327 |
328 | if totalSizeTextCache == -1 || lastCacheTime.Add(refreshCacheEvery).Before(currentTime) {
- 329 | row4 := conn.QueryRowContext(context.Background(), "SELECT SUM(SIZE) FROM pages WHERE contenttype LIKE 'text/%%'")
+ 331 | row4 := conn.QueryRowContext(context.Background(), "SELECT SUM(SIZE) FROM pages WHERE contenttype LIKE 'text/%%' AND (SCHEME = 'gemini' OR SCHEME = 'GEMINI')")
330 | row4.Scan(&totalSizeTextCache)
331 | lastCacheTime = currentTime
332 | }
333 | totalSizeText := totalSizeTextCache
334 | totalSizeText /= 1024 // Bytes to KB
... | ...
350 | Page Count: %d
351 | Capsule Count: %d
352 | Gemsub Feed Count: %d
353 |
354 | Total Size of Geminispace: %.3f GB
- 355 | Total Size of Text Files: %.3f GB (%.2f%% of Geminispace)
+ 357 | Total Size of Text Files within Geminispace: %.3f GB (%.2f%% of Geminispace)
356 |
357 | Number of Domains with SlowDown responses: %d
358 | Number of Domains that responded with an empty META field: %d
359 |
360 | => /search/mimetype/ Mimetypes with Counts
... | ...
437 | handleBacklinks(request, conn, queryUrl)
438 | return
439 | }
440 | })
441 |
+ 444 | // Smallnet search
442 | s.AddRoute("/search/s", func(request sis.Request) {
443 | query, err := request.Query()
444 | if err != nil {
445 | request.TemporaryFailure(err.Error())
446 | return
... | ...
453 | request.SendAbstract("")
454 | return
455 | }
456 |
457 | // Page 1
- 458 | handleSearch(request, conn, query, 1, false)
+ 461 | handleSearch(request, conn, query, 1, false, false, false)
459 | return
460 | }
461 | })
462 |
463 | s.AddRoute("/search/s/:page", func(request sis.Request) {
... | ...
480 | if request.ScrollMetadataRequested {
481 | request.SendAbstract("")
482 | return
483 | }
484 |
- 485 | handleSearch(request, conn, query, page, false)
+ 488 | handleSearch(request, conn, query, page, false, false, false)
+ 489 | return
+ 490 | }
+ 491 | })
+ 492 |
+ 493 | // Geminispace search
+ 494 | s.AddRoute("/search/gemini", func(request sis.Request) {
+ 495 | query, err := request.Query()
+ 496 | if err != nil {
+ 497 | request.TemporaryFailure(err.Error())
+ 498 | return
+ 499 | } else if query == "" {
+ 500 | request.RequestInput("Search Query:")
+ 501 | return
+ 502 | } else {
+ 503 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - '" + query + "'\n"})
+ 504 | if request.ScrollMetadataRequested {
+ 505 | request.SendAbstract("")
+ 506 | return
+ 507 | }
+ 508 |
+ 509 | // Page 1
+ 510 | handleSearch(request, conn, query, 1, false, true, false)
+ 511 | return
+ 512 | }
+ 513 | })
+ 514 |
+ 515 | s.AddRoute("/search/gemini/:page", func(request sis.Request) {
+ 516 | pageStr := request.GetParam("page")
+ 517 | page, err := strconv.Atoi(pageStr)
+ 518 | if err != nil {
+ 519 | request.BadRequest("Couldn't parse int.")
+ 520 | return
+ 521 | }
+ 522 |
+ 523 | query, err := request.Query()
+ 524 | if err != nil {
+ 525 | request.TemporaryFailure(err.Error())
+ 526 | return
+ 527 | } else if query == "" {
+ 528 | request.RequestInput("Search Query:")
+ 529 | return
+ 530 | } else {
+ 531 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - '" + query + "' Page " + pageStr + "\n"})
+ 532 | if request.ScrollMetadataRequested {
+ 533 | request.SendAbstract("")
+ 534 | return
+ 535 | }
+ 536 |
+ 537 | handleSearch(request, conn, query, page, false, true, false)
+ 538 | return
+ 539 | }
+ 540 | })
+ 541 |
+ 542 | // Scroll protocol search
+ 543 | s.AddRoute("/search/scroll", func(request sis.Request) {
+ 544 | query, err := request.Query()
+ 545 | if err != nil {
+ 546 | request.TemporaryFailure(err.Error())
+ 547 | return
+ 548 | } else if query == "" {
+ 549 | request.RequestInput("Search Query:")
+ 550 | return
+ 551 | } else {
+ 552 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - '" + query + "'\n"})
+ 553 | if request.ScrollMetadataRequested {
+ 554 | request.SendAbstract("")
+ 555 | return
+ 556 | }
+ 557 |
+ 558 | // Page 1
+ 559 | handleSearch(request, conn, query, 1, false, false, true)
+ 560 | return
+ 561 | }
+ 562 | })
+ 563 |
+ 564 | s.AddRoute("/search/scroll/:page", func(request sis.Request) {
+ 565 | pageStr := request.GetParam("page")
+ 566 | page, err := strconv.Atoi(pageStr)
+ 567 | if err != nil {
+ 568 | request.BadRequest("Couldn't parse int.")
+ 569 | return
+ 570 | }
+ 571 |
+ 572 | query, err := request.Query()
+ 573 | if err != nil {
+ 574 | request.TemporaryFailure(err.Error())
+ 575 | return
+ 576 | } else if query == "" {
+ 577 | request.RequestInput("Search Query:")
+ 578 | return
+ 579 | } else {
+ 580 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - '" + query + "' Page " + pageStr + "\n"})
+ 581 | if request.ScrollMetadataRequested {
+ 582 | request.SendAbstract("")
+ 583 | return
+ 584 | }
+ 585 |
+ 586 | handleSearch(request, conn, query, page, false, false, true)
486 | return
487 | }
488 | })
489 |
490 | // Debug searching - shows the Score numbers
... | ...
496 | } else if query == "" {
497 | request.RequestInput("Search Query:")
498 | return
499 | } else {
500 | // Page 1
- 501 | handleSearch(request, conn, query, 1, true)
+ 602 | handleSearch(request, conn, query, 1, true, false, false)
502 | return
503 | }
504 | })
505 |
506 | s.AddRoute("/search/debug_s/:page", func(request sis.Request) {
... | ...
517 | return
518 | } else if query == "" {
519 | request.RequestInput("Search Query:")
520 | return
521 | } else {
- 522 | handleSearch(request, conn, query, page, true)
+ 623 | handleSearch(request, conn, query, page, true, false, false)
523 | return
524 | }
525 | })
526 |
527 | s.AddRoute("/search/recent", func(request sis.Request) {
... | ...
629 |
630 | %s
631 | `, builder.String()))
632 | })
633 |
- 634 | s.AddRoute("/search/yearposts", func(request sis.Request) {
- 635 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - Posts From The Past Year\n"})
- 636 | if request.ScrollMetadataRequested {
- 637 | request.SendAbstract("")
- 638 | return
- 639 | }
+ 735 | /*
+ 736 | s.AddRoute("/search/yearposts", func(request sis.Request) {
+ 737 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - Posts From The Past Year\n"})
+ 738 | if request.ScrollMetadataRequested {
+ 739 | request.SendAbstract("")
+ 740 | return
+ 741 | }
640 |
... | ...
636 |
- 641 | page := 1
- 642 | results := 40
- 643 | skip := (page - 1) * results
+ 743 | page := 1
+ 744 | results := 40
+ 745 | skip := (page - 1) * results
644 |
... | ...
640 |
- 645 | pages, totalResultsCount := getPagesWithPublishDateFromLastYear(conn, results, skip)
+ 747 | pages, totalResultsCount := getPagesWithPublishDateFromLastYear(conn, results, skip)
646 |
... | ...
642 |
- 647 | resultsStart := skip + 1
- 648 | resultsEnd := Min(totalResultsCount, skip+results) // + 1 - 1
- 649 | hasNextPage := resultsEnd < totalResultsCount && totalResultsCount != 0
- 650 | hasPrevPage := resultsStart > results
+ 749 | resultsStart := skip + 1
+ 750 | resultsEnd := Min(totalResultsCount, skip+results) // + 1 - 1
+ 751 | hasNextPage := resultsEnd < totalResultsCount && totalResultsCount != 0
+ 752 | hasPrevPage := resultsStart > results
651 |
... | ...
647 |
- 652 | var builder strings.Builder
- 653 | buildPageResults(&builder, pages, false, false)
+ 754 | var builder strings.Builder
+ 755 | buildPageResults(&builder, pages, false, false)
654 |
... | ...
650 |
- 655 | if hasPrevPage {
- 656 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d Previous Page\n", page-1)
- 657 | }
- 658 | if hasNextPage && !hasPrevPage {
- 659 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d/ Next Page\n", page+1)
- 660 | } else if hasNextPage && hasPrevPage {
- 661 | fmt.Fprintf(&builder, "=> /search/yearposts/%d/ Next Page\n", page+1)
- 662 | }
+ 757 | if hasPrevPage {
+ 758 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d Previous Page\n", page-1)
+ 759 | }
+ 760 | if hasNextPage && !hasPrevPage {
+ 761 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d/ Next Page\n", page+1)
+ 762 | } else if hasNextPage && hasPrevPage {
+ 763 | fmt.Fprintf(&builder, "=> /search/yearposts/%d/ Next Page\n", page+1)
+ 764 | }
663 |
... | ...
659 |
- 664 | request.Gemini(fmt.Sprintf(`# Posts From The Past Year
+ 766 | request.Gemini(fmt.Sprintf(`# Posts From The Past Year
665 |
... | ...
661 |
- 666 | => /search/ Home
- 667 | => /search/s/ Search
+ 768 | => /search/ Home
+ 769 | => /search/s/ Search
668 |
... | ...
664 |
- 669 | Note: Currently tries to list only posts that are in English.
+ 771 | Note: Currently tries to list only posts that are in English.
670 |
... | ...
666 |
- 671 | %s
- 672 | `, builder.String()))
- 673 | })
+ 773 | %s
+ 774 | `, builder.String()))
+ 775 | })
674 |
... | ...
670 |
- 675 | s.AddRoute("/search/yearposts/:page", func(request sis.Request) {
- 676 | pageStr := request.GetParam("page")
- 677 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - Posts From The Past Year, Page " + pageStr + "\n"})
- 678 | if request.ScrollMetadataRequested {
- 679 | request.SendAbstract("")
- 680 | return
- 681 | }
+ 777 | s.AddRoute("/search/yearposts/:page", func(request sis.Request) {
+ 778 | pageStr := request.GetParam("page")
+ 779 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - Posts From The Past Year, Page " + pageStr + "\n"})
+ 780 | if request.ScrollMetadataRequested {
+ 781 | request.SendAbstract("")
+ 782 | return
+ 783 | }
682 |
... | ...
678 |
- 683 | page, err := strconv.Atoi(pageStr)
- 684 | if err != nil {
- 685 | request.BadRequest("Couldn't parse int.")
- 686 | return
- 687 | }
+ 785 | page, err := strconv.Atoi(pageStr)
+ 786 | if err != nil {
+ 787 | request.BadRequest("Couldn't parse int.")
+ 788 | return
+ 789 | }
688 |
... | ...
684 |
- 689 | results := 40
- 690 | skip := (page - 1) * results
+ 791 | results := 40
+ 792 | skip := (page - 1) * results
691 |
... | ...
687 |
- 692 | pages, totalResultsCount := getPagesWithPublishDateFromLastYear(conn, results, skip)
+ 794 | pages, totalResultsCount := getPagesWithPublishDateFromLastYear(conn, results, skip)
693 |
... | ...
689 |
- 694 | resultsStart := skip + 1
- 695 | resultsEnd := Min(totalResultsCount, skip+results) // + 1 - 1
- 696 | hasNextPage := resultsEnd < totalResultsCount && totalResultsCount != 0
- 697 | hasPrevPage := resultsStart > results
+ 796 | resultsStart := skip + 1
+ 797 | resultsEnd := Min(totalResultsCount, skip+results) // + 1 - 1
+ 798 | hasNextPage := resultsEnd < totalResultsCount && totalResultsCount != 0
+ 799 | hasPrevPage := resultsStart > results
698 |
... | ...
694 |
- 699 | var builder strings.Builder
- 700 | buildPageResults(&builder, pages, false, false)
+ 801 | var builder strings.Builder
+ 802 | buildPageResults(&builder, pages, false, false)
701 |
... | ...
697 |
- 702 | if hasPrevPage {
- 703 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d Previous Page\n", page-1)
- 704 | }
- 705 | if hasNextPage && !hasPrevPage {
- 706 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d/ Next Page\n", page+1)
- 707 | } else if hasNextPage && hasPrevPage {
- 708 | fmt.Fprintf(&builder, "=> /search/yearposts/%d/ Next Page\n", page+1)
- 709 | }
+ 804 | if hasPrevPage {
+ 805 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d Previous Page\n", page-1)
+ 806 | }
+ 807 | if hasNextPage && !hasPrevPage {
+ 808 | fmt.Fprintf(&builder, "\n=> /search/yearposts/%d/ Next Page\n", page+1)
+ 809 | } else if hasNextPage && hasPrevPage {
+ 810 | fmt.Fprintf(&builder, "=> /search/yearposts/%d/ Next Page\n", page+1)
+ 811 | }
710 |
... | ...
706 |
- 711 | request.Gemini(fmt.Sprintf(`# Posts From The Past Year
+ 813 | request.Gemini(fmt.Sprintf(`# Posts From The Past Year
712 |
... | ...
708 |
- 713 | => /search/ Home
- 714 | => /search/s/ Search
+ 815 | => /search/ Home
+ 816 | => /search/s/ Search
715 |
... | ...
711 |
- 716 | Note: Currently tries to list only posts that are in English.
+ 818 | Note: Currently tries to list only posts that are in English.
717 |
... | ...
713 |
- 718 | %s
- 719 | `, builder.String()))
- 720 | })
+ 820 | %s
+ 821 | `, builder.String()))
+ 822 | })
+ 823 | */
721 |
722 | s.AddRoute("/search/audio", func(request sis.Request) {
723 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Abstract: "# AuraGem Search - Indexed Audio Files\n"})
724 | if request.ScrollMetadataRequested {
725 | request.SendAbstract("")
... | ...
1079 |
1080 | %s
1081 | `, url.String(), builder.String()))
1082 | }
1083 |
- 1084 | func handleSearch(request sis.Request, conn *sql.DB, query string, page int, showScores bool) {
+ 1187 | func handleSearch(request sis.Request, conn *sql.DB, query string, page int, showScores bool, gemini_only bool, scroll_only bool) {
1085 | //rawQuery := c.URL().RawQuery
1086 | rawQuery, err := request.RawQuery()
1087 | if err != nil {
1088 | request.TemporaryFailure(err.Error())
1089 | return
... | ...
1100 | queryFiltered = strings.Replace(queryFiltered, "Project Gemini", "\"Project Gemini\"", 1)
1101 | queryFiltered = strings.Replace(queryFiltered, "project Gemini", "\"project Gemini\"", 1)
1102 | queryFiltered = strings.Replace(queryFiltered, "Project gemini", "\"Project gemini\"", 1)
1103 | //queryFiltered = strings.Replace(queryFiltered, "gemini", "\"gemini protocol\"", 1) // TODO: Doesn't work well yet
1104 |
- 1105 | actualQuery := strings.Replace(fts_searchQuery, `%%query%%`, queryFiltered, 2) // TODO: Support for protocol-specific searching.
+ 1208 | actualQuery := ""
+ 1209 | if !gemini_only && !scroll_only {
+ 1210 | actualQuery = strings.Replace(fts_searchQuery, `%%query%%`, queryFiltered, 2) // TODO: Support for protocol-specific searching.
+ 1211 | } else if gemini_only {
+ 1212 | actualQuery = strings.Replace(fts_searchQuery_protocol, `%%query%%`, queryFiltered, 2) // TODO: Support for protocol-specific searching.
+ 1213 | actualQuery = strings.Replace(actualQuery, `%%protocol%%`, "gemini", 1)
+ 1214 | } else if scroll_only {
+ 1215 | actualQuery = strings.Replace(fts_searchQuery_protocol, `%%query%%`, queryFiltered, 2) // TODO: Support for protocol-specific searching.
+ 1216 | actualQuery = strings.Replace(actualQuery, `%%protocol%%`, "scroll", 1)
+ 1217 | }
1106 | actualQuery = strings.Replace(actualQuery, `%%first%%`, strconv.Itoa(results), 1)
1107 | actualQuery = strings.Replace(actualQuery, `%%skip%%`, strconv.Itoa(skip), 1)
1108 |
1109 | parts := strings.Split(queryFiltered, " ")
1110 | var matchesBuilder strings.Builder
... | ...
1133 | var totalResultsCount = 0 // Total count of all results, regardless of pagination
1134 | if rows_err == nil {
1135 | defer rows.Close()
1136 | for rows.Next() {
1137 | var page Page
- 1138 | scan_err := rows.Scan(&totalResultsCount, &page.Score, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 1250 | scan_err := rows.Scan(&totalResultsCount, &page.Score, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
1139 | if scan_err == nil {
1140 | pages = append(pages, page)
1141 | } else {
1142 | prevPage := Page{}
1143 | if len(pages) > 0 {
... | ...
1179 | request.Gemini("\nNote that AuraGem Search does not ensure or rank based on the popularity or accuracy of the information within any of the pages listed in these search results. One cannot presume that information published within Geminispace is or is not for ill-intent or misinformation, even if it's popular or well-linked, so one must use their best judgement in determining the trustworthiness of such content themselves.\n")
1180 | }
1181 |
1182 | func handleSearchIndex(request sis.Request, conn *sql.DB) {
1183 | request.Gemini("Test\n")
- 1184 | query := "SELECT FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, P.ID, P.URL, P.SCHEME, P.DOMAINID, P.CONTENTTYPE, P.CHARSET, P.LANGUAGE, P.LINECOUNT, P.TITLE, P.PROMPT, P.SIZE, P.HASH, P.FEED, P.PUBLISHDATE, P.INDEXTIME, P.ALBUM, P.ARTIST, P.ALBUMARTIST, P.COMPOSER, P.TRACK, P.DISC, P.COPYRIGHT, P.CRAWLINDEX, P.DATE_ADDED, P.LAST_SUCCESSFUL_VISIT, P.HIDDEN FROM PAGES P"
+ 1296 | query := "SELECT FIRST %%first%% SKIP %%skip%% COUNT(*) OVER () totalCount, P.ID, P.URL, P.SCHEME, P.DOMAINID, P.CONTENTTYPE, P.CHARSET, P.LANGUAGE, P.LINECOUNT, P.UDC, P.TITLE, P.PROMPT, P.SIZE, P.HASH, P.FEED, P.PUBLISHDATE, P.INDEXTIME, P.ALBUM, P.ARTIST, P.ALBUMARTIST, P.COMPOSER, P.TRACK, P.DISC, P.COPYRIGHT, P.CRAWLINDEX, P.DATE_ADDED, P.LAST_SUCCESSFUL_VISIT, P.HIDDEN FROM PAGES P"
1185 | results_per_query := 10
1186 | current_query_index := 1
1187 | max_results := 100000000 // TODO
1188 | first := true
1189 |
... | ...
1202 | var totalResultsCount = 0 // Total count of all results, regardless of pagination
1203 | if rows_err == nil {
1204 | defer rows.Close()
1205 | for rows.Next() {
1206 | var page Page
- 1207 | scan_err := rows.Scan(&totalResultsCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 1319 | scan_err := rows.Scan(&totalResultsCount, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
1208 | if scan_err == nil {
1209 | pages = append(pages, page)
1210 | } else {
1211 | panic(scan_err)
1212 | }
... | ...
1262 | var totalResultsCount = 0 // Total count of all results, regardless of pagination
1263 | if rows_err == nil {
1264 | defer rows.Close()
1265 | for rows.Next() {
1266 | var page Page
- 1267 | scan_err := rows.Scan(&totalResultsCount, &page.Score, &page.Highlight, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
+ 1379 | scan_err := rows.Scan(&totalResultsCount, &page.Score, &page.Highlight, &page.Id, &page.Url, &page.Scheme, &page.DomainId, &page.Content_type, &page.Charset, &page.Language, &page.Linecount, &page.Udc, &page.Title, &page.Prompt, &page.Size, &page.Hash, &page.Feed, &page.PublishDate, &page.Index_time, &page.Album, &page.Artist, &page.AlbumArtist, &page.Composer, &page.Track, &page.Disc, &page.Copyright, &page.CrawlIndex, &page.Date_added, &page.LastSuccessfulVisit, &page.Hidden)
1268 | if scan_err == nil {
1269 | pages = append(pages, page)
1270 | } else {
1271 | panic(scan_err)
1272 | }
gemini/search/types.go
... | ...
37 |
38 | Content_type string
39 | Charset string
40 | Language string
41 | Linecount int
+ 42 | Udc string
42 |
43 | Title string // Used for text/gemini and text/markdown files with page titles
44 | // content []u8 // TODO
45 | Prompt string // For input prompt urls
46 | Headings string // Empty unless specifically queried for as we don't want to query this from the DB due to potential large size
gemini/starwars/starwars.go
... | ...
22 | request.Redirect("/starwars2/")
23 | })
24 |
25 | s.AddRoute("/starwars2/", func(request sis.Request) {
26 | updateDate, _ := time.ParseInLocation(time.RFC3339, "2024-03-19T08:23:00", time.Local)
- 27 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# Star Wars Database\n"})
+ 27 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: updateDate, Language: "en", Abstract: "# Star Wars Database\n"})
28 | if request.ScrollMetadataRequested {
29 | request.SendAbstract("")
30 | return
31 | }
32 |
... | ...
65 | handleMoviesCSV(request, conn, false)
66 | })
67 |
68 | s.AddRoute("/starwars2/timeline/shows", func(request sis.Request) {
69 | shows, lastUpdate := GetShows(conn)
- 70 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: TV Shows (Timeline)\n"})
+ 70 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: TV Shows (Timeline)\n"})
71 | if request.ScrollMetadataRequested {
72 | request.SendAbstract("")
73 | return
74 | }
75 |
... | ...
101 | oneshots, lastUpdate2 := GetComicOneshots(conn, true)
102 | if lastUpdate.Before(lastUpdate2) {
103 | lastUpdate = lastUpdate2
104 | }
105 |
- 106 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comics (Timeline)\n"})
+ 106 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comics (Timeline)\n"})
107 | if request.ScrollMetadataRequested {
108 | request.SendAbstract("")
109 | return
110 | }
111 |
... | ...
153 | oneshots, lastUpdate2 := GetComicOneshots(conn, false)
154 | if lastUpdate.Before(lastUpdate2) {
155 | lastUpdate = lastUpdate2
156 | }
157 |
- 158 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comics (Publication)\n"})
+ 158 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comics (Publication)\n"})
159 | if request.ScrollMetadataRequested {
160 | request.SendAbstract("")
161 | return
162 | }
163 |
... | ...
192 | `, builder.String()))
193 | })
194 |
195 | s.AddRoute("/starwars2/timeline/comics/tpbs", func(request sis.Request) {
196 | tpbs, lastUpdate := GetTPBs(conn, true)
- 197 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic TPBs (Timeline)\n"})
+ 197 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic TPBs (Timeline)\n"})
198 | if request.ScrollMetadataRequested {
199 | request.SendAbstract("")
200 | return
201 | }
202 |
... | ...
217 | `, builder.String()))
218 | })
219 |
220 | s.AddRoute("/starwars2/publication/comics/tpbs", func(request sis.Request) {
221 | tpbs, lastUpdate := GetTPBs(conn, false)
- 222 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic TPBs (Publication)\n"})
+ 222 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic TPBs (Publication)\n"})
223 | if request.ScrollMetadataRequested {
224 | request.SendAbstract("")
225 | return
226 | }
227 |
... | ...
242 | `, builder.String()))
243 | })
244 |
245 | s.AddRoute("/starwars2/timeline/comics/issues", func(request sis.Request) {
246 | issues, lastUpdate := GetComicIssues(conn, true)
- 247 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic Issues (Timeline)\n"})
+ 247 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic Issues (Timeline)\n"})
248 | if request.ScrollMetadataRequested {
249 | request.SendAbstract("")
250 | return
251 | }
252 |
... | ...
267 | `, builder.String()))
268 | })
269 |
270 | s.AddRoute("/starwars2/publication/comics/issues", func(request sis.Request) {
271 | issues, lastUpdate := GetComicIssues(conn, false)
- 272 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic Issues (Publication)\n"})
+ 272 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Comic Issues (Publication)\n"})
273 | if request.ScrollMetadataRequested {
274 | request.SendAbstract("")
275 | return
276 | }
277 |
... | ...
302 |
303 | standalones, lastUpdate2 := GetBookStandalones(conn)
304 | if lastUpdate.Before(lastUpdate2) {
305 | lastUpdate = lastUpdate2
306 | }
- 307 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Book Series (Timeline)\n"})
+ 307 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Book Series (Timeline)\n"})
308 | if request.ScrollMetadataRequested {
309 | request.SendAbstract("")
310 | return
311 | }
312 |
... | ...
324 | `, builder.String()))
325 | })
326 |
327 | s.AddRoute("/starwars2/timeline/books", func(request sis.Request) {
328 | books, lastUpdate := GetBooks(conn)
- 329 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Books (Timeline)\n"})
+ 329 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Books (Timeline)\n"})
330 | if request.ScrollMetadataRequested {
331 | request.SendAbstract("")
332 | return
333 | }
334 |
... | ...
349 | })
350 | }
351 |
352 | func handleMovies(request sis.Request, conn *sql.DB, timeline bool) {
353 | movies, lastUpdate := GetMovies(conn, timeline)
- 354 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Movies\n"})
+ 354 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Movies\n"})
355 | if request.ScrollMetadataRequested {
356 | request.SendAbstract("")
357 | return
358 | }
359 |
... | ...
372 | `, builder.String()))
373 | }
374 |
375 | func handleMoviesCSV(request sis.Request, conn *sql.DB, timeline bool) {
376 | movies, lastUpdate := GetMovies(conn, timeline)
- 377 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Movies CSV\n"})
+ 377 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, Author: "Christian Lee Seibold", PublishDate: publishDate, UpdateDate: lastUpdate, Language: "en", Abstract: "# Star Wars Database: Movies CSV\n"})
378 | if request.ScrollMetadataRequested {
379 | request.SendAbstract("text/csv")
380 | return
381 | }
382 |
gemini/texts/christianity/christianity.go
... | ...
93 | // Cache the books from the ASV version of the bible. These should be the same for the ESV bible as well. Note: This does not include the apocrypha.
94 | asvBooks := GetBooks(englishBibleVersions[0].Id, apiKey)
95 |
96 | g.AddRoute("/scriptures/christian/", func(request sis.Request) {
97 | request.SetNoLanguage()
+ 98 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
98 | var builder strings.Builder
99 | fmt.Fprintf(&builder, "## Bible Versions\n")
100 | fmt.Fprintf(&builder, "### English\n")
101 | fmt.Fprintf(&builder, "=> /scriptures/christian/bible/esv/ ESV Bible\n")
102 | for _, version := range englishBibleVersions {
... | ...
159 |
160 | => https://scripture.api.bible Bibles Powered by API.Bible
161 |
162 | %s
163 |
- 164 | Tags: #bible #new #old #testament #septuagint #pentateuch
+ 165 | Tags: #bible #new #old #testament #septuagint #pentateuch
165 | `, builder.String()))
166 | })
167 |
168 | g.AddRoute("/scriptures/christian/bible/esv/", func(request sis.Request) {
169 | request.SetLanguage("en-US")
... | ...
165 | `, builder.String()))
166 | })
167 |
168 | g.AddRoute("/scriptures/christian/bible/esv/", func(request sis.Request) {
169 | request.SetLanguage("en-US")
+ 171 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
170 | var builder strings.Builder
171 | for _, book := range asvBooks {
172 | fmt.Fprintf(&builder, "=> /scriptures/christian/bible/esv/%s/ %s\n", url.PathEscape(book.Name+" 1"), book.Name)
173 | }
174 |
... | ...
186 | `, builder.String()))
187 | })
188 |
189 | g.AddRoute("/scriptures/christian/bible/esv/:text", func(request sis.Request) {
190 | request.SetLanguage("en-US")
+ 193 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
191 | text := request.GetParam("text")
192 | resp := GetPassages(text)
193 | var builder strings.Builder
194 | for _, s := range resp.Passages {
195 | fmt.Fprintf(&builder, "%s", s)
... | ...
203 |
204 | g.AddRoute("/scriptures/christian/bible/:id", func(request sis.Request) {
205 | versionId := request.GetParam("id")
206 | version := GetBibleVersion(versionId, apiKey)
207 | request.SetLanguage(version.Language.Id)
+ 211 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
208 | books := GetBooks(versionId, apiKey)
209 | var builder strings.Builder
210 | for _, book := range books {
211 | fmt.Fprintf(&builder, "=> /scriptures/christian/bible/%s/%s/ %s\n", versionId, book.Id, book.Name)
212 | }
... | ...
226 | g.AddRoute("/scriptures/christian/bible/:id/:book", func(request sis.Request) {
227 | versionId := request.GetParam("id")
228 | bookId := request.GetParam("book")
229 | version := GetBibleVersion(versionId, apiKey)
230 | request.SetLanguage(version.Language.Id)
+ 235 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
231 | book := GetBook(versionId, bookId, apiKey, true)
232 | var builder strings.Builder
233 | for _, chapter := range book.Chapters {
234 | fmt.Fprintf(&builder, "=> /scriptures/christian/bible/%s/chapter/%s/ Chapter %s\n", versionId, chapter.Id, chapter.Number)
235 | }
... | ...
248 | g.AddRoute("/scriptures/christian/bible/:id/chapter/:chapter", func(request sis.Request) {
249 | versionId := request.GetParam("id")
250 | chapterId := request.GetParam("chapter")
251 | version := GetBibleVersion(versionId, apiKey)
252 | request.SetLanguage(version.Language.Id)
+ 258 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
253 | chapter := GetChapter(versionId, chapterId, apiKey)
254 | var builder strings.Builder
255 | fmt.Fprintf(&builder, "%s", chapter.Content)
256 | /*for _, chapter := range book.Chapters {
257 | fmt.Fprintf(&builder, "=> /scriptures/christian/bible/%s/%s/%s Chapter %s\n", versionId, book.Id, chapter.Id, chapter.Number)
gemini/texts/islam/islam.go
... | ...
107 |
108 | versionNames["arabic"] = "Qur'an"
109 |
110 | g.AddRoute("/scriptures/islam", func(request sis.Request) {
111 | request.SetNoLanguage()
+ 112 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
112 | var builder strings.Builder
113 | fmt.Fprintf(&builder, "## Qur'an Versions\n\n=> /scriptures/islam/quran/arabic/ Arabic\n")
114 | fmt.Fprintf(&builder, "### English\n")
115 | for _, version := range englishQuranVersions {
116 | if version.Identifier != "" {
... | ...
164 |
165 | => https://alquran.cloud/ Powered by Al Quran Cloud
166 |
167 | %s
168 |
- 169 | Tags: #quran #qur'an #koran #القرآن
+ 170 | Tags: #quran #qur'an #koran #القرآن
170 | `, builder.String()))
171 | })
172 |
173 | g.AddRoute("/scriptures/islam/quran/:version", func(request sis.Request) {
... | ...
169 | `, builder.String()))
170 | })
171 |
172 | g.AddRoute("/scriptures/islam/quran/:version", func(request sis.Request) {
+ 175 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
174 | versionId := request.GetParam("version")
175 | var builder strings.Builder
176 | for _, surah := range quranSurahs {
177 | fmt.Fprintf(&builder, "=> /scriptures/islam/quran/%s/%d/ Surah %d: %s (%s)\n", versionId, surah.Number, surah.Number, surah.EnglishNameTranslation, surah.EnglishName)
178 | }
... | ...
191 | versionId := request.GetParam("version")
192 | surahNumber := request.GetParam("surah")
193 |
194 | surah := GetSurah(versionId, surahNumber)
195 | request.SetLanguage(surah.Edition.Language)
+ 198 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
196 |
197 | var builder strings.Builder
198 | for _, ayah := range surah.Ayahs {
199 | fmt.Fprintf(&builder, "[%d] %s\n", ayah.NumberInSurah, ayah.Text)
200 | }
gemini/texts/judaism/judaism.go
... | ...
44 | handleText(ref, request)
45 | })
46 | }
47 |
48 | func handleIndex(index []SefariaIndexCategoryOrText, request sis.Request) {
+ 49 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
49 | var builder strings.Builder
50 | for _, category := range index {
51 | fmt.Fprintf(&builder, "=> /scriptures/jewish/?%s %s\n", url.QueryEscape(category.Category), category.Category)
52 | }
53 |
... | ...
84 | `
85 | request.Gemini(fmt.Sprintf(template, builder.String()))
86 | }
87 |
88 | func handleCategory(index []SefariaIndexCategoryOrText, query string, request sis.Request) {
+ 90 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
89 | categories := strings.Split(query, "/")
90 | var categoryStringBuilder strings.Builder
91 | for i, c := range categories {
92 | if i == 0 {
93 | fmt.Fprintf(&categoryStringBuilder, "%s ", c)
... | ...
119 | `, categoryStringBuilder.String(), builder.String()))
120 | }
121 |
122 | func handleText(ref string, request sis.Request) {
123 | text := GetText(ref, "", "" /*"Tanakh: The Holy Scriptures, published by JPS"*/)
+ 126 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
124 |
125 | var startingChapterInRef int = 1
126 | if text.TextDepth-1 >= 0 && text.TextDepth-1 <= len(text.Sections) {
127 | if utf8.Valid(text.Sections[text.TextDepth-2]) {
128 | i, err := strconv.Atoi(string(text.Sections[text.TextDepth-2]))
gemini/texts/texts.go
... | ...
16 | sis "gitlab.com/clseibold/smallnetinformationservices"
17 | )
18 |
19 | func HandleTexts(g sis.ServerHandle) {
20 | g.AddRoute("/scriptures/", func(request sis.Request) {
+ 21 | request.SetClassification(sis.ScrollResponseUDC_Scripture)
21 | request.Gemini(`# Religious Texts
22 |
23 | => /scriptures/jewish/ ✡ Jewish Texts
24 | => /scriptures/christian/ ✝ Christian Texts
25 | => /scriptures/islam/ ☪ Islamic Texts
gemini/weather.go
... | ...
18 | publishDate, _ := time.ParseInLocation(time.RFC3339, "2024-03-19T13:51:00", time.Local)
19 | g.AddRoute("/weather", func(request sis.Request) {
20 | request.Redirect("/weather/")
21 | })
22 | g.AddRoute("/weather/", func(request sis.Request) {
- 23 | request.SetScrollMetadataResponse(sis.ScrollMetadata{PublishDate: publishDate, UpdateDate: time.Now(), Language: "en", Abstract: "# Weather\n"})
+ 23 | request.SetScrollMetadataResponse(sis.ScrollMetadata{Classification: sis.ScrollResponseUDC_Reference, PublishDate: publishDate, UpdateDate: time.Now(), Language: "en", Abstract: "# Weather\n"})
24 | if request.ScrollMetadataRequested {
25 | _ = request.SendAbstract("")
26 | return
27 | }
28 | iqAirResponse := getNearestLocation(request)
gemini/youtube/youtube.go
... | ...
102 | }
103 | }
104 | }
105 | }
106 |
+ 107 | func handleVideoClassification(video *youtube.Video, request sis.Request) {
+ 108 | handleTopicId := false
+ 109 | switch video.Snippet.CategoryId {
+ 110 | case "1": // Film & Animation
+ 111 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 112 | case "2": // Autos & Vehicles
+ 113 | request.SetClassification(sis.ScrollResponseUDC_Engineering)
+ 114 | case "10": // Music
+ 115 | request.SetClassification(sis.ScrollResponseUDC_Music)
+ 116 | case "15": // Pets and Animals
+ 117 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 118 | case "17": // Sports
+ 119 | request.SetClassification(sis.ScrollResponseUDC_Sport)
+ 120 | case "18": // Short Movies
+ 121 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 122 | case "19": // Travel & Events
+ 123 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 124 | case "20": // Gaming
+ 125 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 126 | case "21": // Videoblogging
+ 127 | request.SetClassification(sis.ScrollResponseUDC_PersonalLog)
+ 128 | case "22": // People & Blogs
+ 129 | request.SetClassification(sis.ScrollResponseUDC_PersonalLog)
+ 130 | case "23": // Comedy
+ 131 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 132 | case "24": // Entertainment
+ 133 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 134 | case "25": // News and Politics
+ 135 | request.SetClassification(sis.ScrollResponseUDC_SocialScience)
+ 136 | case "26": // Howto and Style
+ 137 | request.SetClassification(sis.ScrollResponseUDC_Reference)
+ 138 | case "27": // Education
+ 139 | request.SetClassification(sis.ScrollResponseUDC_SocialScience)
+ 140 | case "28": // Science and Technology
+ 141 | handleTopicId = true
+ 142 | //request.SetClassification(sis.ScrollResponseUDC_Technology) // TODO
+ 143 | case "29": // Nonprofits & Activism
+ 144 | request.SetClassification(sis.ScrollResponseUDC_SocialScience)
+ 145 | case "30": // Movies
+ 146 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 147 | case "31": // Anime/Animation
+ 148 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 149 | case "32": // Action/Adventure
+ 150 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 151 | case "33": // Classics
+ 152 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 153 | case "34": // Comedy
+ 154 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 155 | case "35": // Documentary
+ 156 | handleTopicId = true
+ 157 | case "36": // Drama
+ 158 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 159 | case "37": // Family
+ 160 | request.SetClassification(sis.ScrollResponseUDC_PersonalLog)
+ 161 | case "38": // Foreign
+ 162 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 163 | case "39": // Horror
+ 164 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 165 | case "40": // Sci-Fi/Fantasy
+ 166 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 167 | case "41": // Thriller
+ 168 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 169 | case "42": // Shorts
+ 170 | handleTopicId = true
+ 171 | case "43": // Shows
+ 172 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 173 | case "44": // Trailers
+ 174 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 175 | }
+ 176 |
+ 177 | if handleTopicId {
+ 178 | outer:
+ 179 | for _, topic := range video.TopicDetails.TopicIds {
+ 180 | switch topic {
+ 181 | case "/m/01k8wb":
+ 182 | request.SetClassification(sis.ScrollResponseUDC_GeneralKnowledge)
+ 183 | break outer
+ 184 | case "/m/04rlf", "/m/02mscn", "/m/0ggq0m", "/m/01lyv", "/m/02lkt", "/m/0glt670", "/m/05rwpb", "/m/03_d0", "/m/028sqc", "/m/0g293", "/m/064t9", "/m/06cqb", "/m/06j6l", "/m/06by7", "/m/0gywn":
+ 185 | request.SetClassification(sis.ScrollResponseUDC_Music)
+ 186 | break outer
+ 187 | case "/m/0bzvm2", "/m/025zzc", "/m/02ntfj", "/m/0b1vjn", "/m/02hygl", "/m/04q1x3q", "/m/01sjng", "/m/0403l3g", "/m/021bp2", "/m/022dc6", "/m/03hf_rm": // Gaming
+ 188 | request.SetClassification(sis.ScrollResponseUDC_GamingVideos)
+ 189 | break outer
+ 190 | case "/m/06ntj", "/m/0jm_", "/m/018jz", "/m/018w8", "/m/01cgz", "/m/09xp_", "/m/02vx4", "/m/037hz", "/m/03tmr", "/m/01h7lh", "/m/0410tth", "/m/07bs0", "/m/07_53": // Sports
+ 191 | request.SetClassification(sis.ScrollResponseUDC_Sport)
+ 192 | break outer
+ 193 | case "/m/02jjt", "/m/09kqc", "/m/02vxn", "/m/05qjc", "/m/066wd", "/m/0f2f9": // Entertainment
+ 194 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 195 | break outer
+ 196 | case "/m/032tl": // Fashion -> Art
+ 197 | request.SetClassification(sis.ScrollResponseUDC_Art)
+ 198 | break outer
+ 199 | case "/m/027x7n": // Fitness -> Sport
+ 200 | request.SetClassification(sis.ScrollResponseUDC_Sport)
+ 201 | break outer
+ 202 | case "/m/02wbm": // Food -> Art
+ 203 | request.SetClassification(sis.ScrollResponseUDC_Art)
+ 204 | break outer
+ 205 | case "/m/03glg": // Hobby -> Recreation
+ 206 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 207 | break outer
+ 208 | case "/m/068hy": // Pets -> Recreation
+ 209 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 210 | break outer
+ 211 | case "/m/041xxh": // Beauty
+ 212 | request.SetClassification(sis.ScrollResponseUDC_Art)
+ 213 | break outer
+ 214 | case "/m/07c1v": // Computer Technology
+ 215 | request.SetClassification(sis.ScrollResponseUDC_Class0)
+ 216 | break outer
+ 217 | case "/m/07bxq": // Tourism -> Recreation
+ 218 | request.SetClassification(sis.ScrollResponseUDC_Entertainment)
+ 219 | break outer
+ 220 | case "/m/07yv9": // Vehicles -> Engineering/General Technology
+ 221 | request.SetClassification(sis.ScrollResponseUDC_Engineering)
+ 222 | break outer
+ 223 | case "/m/06bvp": // Religion
+ 224 | request.SetClassification(sis.ScrollResponseUDC_Religion)
+ 225 | break outer
+ 226 | case "/m/05qt0": // Politics
+ 227 | request.SetClassification(sis.ScrollResponseUDC_SocialScience)
+ 228 | break outer
+ 229 | case "/m/01h6rj": // Military
+ 230 | request.SetClassification(sis.ScrollResponseUDC_SocialScience)
+ 231 | break outer
+ 232 | case "/m/0kt51": // Health
+ 233 | request.SetClassification(sis.ScrollResponseUDC_Medicine)
+ 234 | break outer
+ 235 | case "/m/09s1f": // Business
+ 236 | request.SetClassification(sis.ScrollResponseUDC_AppliedScience)
+ 237 | break outer
+ 238 | case "/m/098wr", "/m/019_rr":
+ 239 | request.SetClassification(sis.ScrollResponseUDC_SocialScience)
+ 240 | }
+ 241 | }
+ 242 | }
+ 243 | }
+ 244 |
107 | func getVideoPageRouteFunc(service *youtube.Service) sis.RequestHandler {
108 | return func(request sis.Request) {
109 | id := request.GetParam("id")
110 | call := service.Videos.List([]string{"id", "snippet", "status"}).Id(id).MaxResults(1)
111 | response, err := call.Do()
... | ...
117 | if len(response.Items) == 0 {
118 | request.TemporaryFailure("Video not found.")
119 | return
120 | }
121 | video := response.Items[0]
+ 260 | handleVideoClassification(video, request)
122 |
123 | lang := request.Server.DefaultLanguage()
124 | if video.Snippet.DefaultLanguage != "" {
125 | lang = video.Snippet.DefaultLanguage
126 | }
go.mod
... | ...
3 | go 1.22.0
4 |
5 | require (
6 | git.sr.ht/~adnano/go-gemini v0.2.4
7 | github.com/clseibold/go-gemini v0.0.0-20240314051634-436d3e54df5c
- 8 | github.com/dhowden/tag v0.0.0-20240122214204-713ab0e94639
+ 8 | github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
9 | github.com/efarrer/iothrottler v0.0.3
10 | github.com/gammazero/deque v0.2.1
11 | github.com/go-stack/stack v1.8.1
12 | github.com/google/go-github/v60 v60.0.0
13 | github.com/juju/ratelimit v1.0.2
... | ...
15 | github.com/krayzpipes/cronticker v0.0.1
16 | github.com/nakagami/firebirdsql v0.9.8
17 | github.com/rivo/uniseg v0.4.7
18 | github.com/rs/zerolog v1.32.0
19 | github.com/spf13/cobra v1.8.0
- 20 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240324210426-e7aa5799c6a1
+ 20 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240519013546-ba625b844724
21 | golang.org/x/net v0.22.0
22 | golang.org/x/oauth2 v0.18.0
23 | golang.org/x/text v0.14.0
24 | golang.org/x/time v0.5.0
25 | google.golang.org/api v0.170.0
go.sum
... | ... 206 | github.com/warpfork/go-fsx v0.4.0 h1:mlSH89UOECT5+NdRo8gPaE92Pm1xvt6cbzGkFa4QcsA= 207 | github.com/warpfork/go-fsx v0.4.0/go.mod h1:oTACCMj+Zle+vgVa5SAhGAh7WksYpLgGUCKEAVc+xPg= 208 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 209 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240324210426-e7aa5799c6a1 h1:9m9kPa2AQGY6x+u54ytHKQBdv3XMzxiBDSL6x3jcprw= 210 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240324210426-e7aa5799c6a1/go.mod h1:1K8FkDn3Ke4DQ+RTO5884rtfXjIquI5Foy5lfRTO9x4= + 211 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240518212713-19da36f6a3b0 h1:rUTaZGpzvr5KHkluGFGCb+L8x+KlWVHLk1Bt9HxYEMg= + 212 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240518212713-19da36f6a3b0/go.mod h1:f0q6bWhDSyXUpREpLfqBGg3u62Jckf4PwstTP5JTwhk= + 213 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240518230856-7e615058f8da h1:l7a/MwHRaBsPL7QtMbcizP1azuCTYoVnPxmow453e9o= + 214 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240518230856-7e615058f8da/go.mod h1:f0q6bWhDSyXUpREpLfqBGg3u62Jckf4PwstTP5JTwhk= + 215 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240518232455-7460f6397396 h1:Z/wysUmHUoULJtFEDXJpJZjCRP4Q2pU2VmrRak9WNiM= + 216 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240518232455-7460f6397396/go.mod h1:f0q6bWhDSyXUpREpLfqBGg3u62Jckf4PwstTP5JTwhk= + 217 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240519011725-99e1ab573d68 h1:Wzhli9AztEqX90/vpe2MkO3fWpUUOIx+eLKdzgGbyQM= + 218 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240519011725-99e1ab573d68/go.mod h1:f0q6bWhDSyXUpREpLfqBGg3u62Jckf4PwstTP5JTwhk= + 219 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240519013546-ba625b844724 h1:MtbHE0H5WHEgv3lBWiD03sOVrjgqq0j5D+PwA0u18hA= + 220 | gitlab.com/clseibold/smallnetinformationservices v0.0.0-20240519013546-ba625b844724/go.mod h1:f0q6bWhDSyXUpREpLfqBGg3u62Jckf4PwstTP5JTwhk= 211 | gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= 212 | gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= 213 | go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= 214 | go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= 215 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
migration/migrations/2021-06-08T140237Z_SearchInitial.go
... | ...
61 |
62 | contenttype character varying(250) COLLATE UNICODE,
63 | charset character varying(250) COLLATE UNICODE,
64 | language character varying(250) COLLATE UNICODE,
65 | linecount integer,
+ 66 | udc character varying(25) NOT NULL COLLATE UNICODE_CI,
66 |
67 | title character varying(250) NOT NULL COLLATE UNICODE_CI,
68 | prompt character varying(250) NOT NULL COLLATE UNICODE_CI,
69 | size integer NOT NULL,
70 | hash character varying(250) NOT NULL,
stats_tool/main.go (created)
+ 1 | package main
+ 2 |
+ 3 | func main() {
+ 4 |
+ 5 | }