2019-05-07 20:51:21 +02:00
#!/usr/bin/env python3
2019-04-17 03:58:00 +02:00
# GAnarchy - project homepage generator
# Copyright (C) 2019 Soni L.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import sqlite3
import click
import os
import subprocess
import hashlib
2019-05-19 15:17:04 +02:00
import hmac
2019-04-17 03:58:00 +02:00
import jinja2
2019-05-28 02:14:09 +02:00
import re
2019-06-20 02:49:27 +02:00
import qtoml
from collections import defaultdict
from urllib . parse import urlparse
2019-04-17 03:58:00 +02:00
2019-05-28 02:14:09 +02:00
MIGRATIONS = {
2019-06-20 02:49:27 +02:00
" toml-config " : (
2019-05-28 02:14:09 +02:00
(
2019-06-20 02:49:27 +02:00
''' UPDATE " repo_history " SET " project " = (SELECT " git_commit " FROM " config " ) WHERE " project " IS NULL ''' ,
''' ALTER TABLE " repos " RENAME TO " repos_old " ''' , ) ,
(
''' UPDATE " repo_history " SET " project " = NULL WHERE " project " = (SELECT " git_commit " FROM " config " ) ''' ,
''' ALTER TABLE " repos_old " RENAME TO " repos " ''' , ) ,
" switches to toml config format. the old ' repos ' table is preserved as ' repos_old ' "
2019-05-28 02:14:09 +02:00
) ,
" better-project-management " : (
(
2019-06-20 02:49:27 +02:00
''' ALTER TABLE " repos " ADD COLUMN " branch " TEXT ''' ,
''' ALTER TABLE " repos " ADD COLUMN " project " TEXT ''' ,
''' CREATE UNIQUE INDEX " repos_url_branch_project " ON " repos " ( " url " , " branch " , " project " ) ''' ,
''' CREATE INDEX " repos_project " ON " repos " ( " project " ) ''' ,
''' ALTER TABLE " repo_history " ADD COLUMN " branch " TEXT ''' ,
''' ALTER TABLE " repo_history " ADD COLUMN " project " TEXT ''' ,
''' CREATE INDEX " repo_history_url_branch_project " ON " repo_history " ( " url " , " branch " , " project " ) ''' , ) ,
2019-05-28 02:14:09 +02:00
(
2019-06-20 02:49:27 +02:00
''' DELETE FROM " repos " WHERE " branch " IS NOT NULL OR " project " IS NOT NULL ''' ,
''' DELETE FROM " repo_history " WHERE " branch " IS NOT NULL OR " project " IS NOT NULL ''' , ) ,
2019-05-28 02:14:09 +02:00
" supports multiple projects, and allows choosing non-default branches "
) ,
" test " : (
2019-06-20 02:49:27 +02:00
( ''' -- apply ''' , ) ,
( ''' -- revert ''' , ) ,
2019-05-28 02:14:09 +02:00
" does nothing "
)
}
data_home = os . environ . get ( ' XDG_DATA_HOME ' , ' ' )
if not data_home :
data_home = os . environ [ ' HOME ' ] + ' /.local/share '
data_home = data_home + " /ganarchy "
cache_home = os . environ . get ( ' XDG_CACHE_HOME ' , ' ' )
if not cache_home :
cache_home = os . environ [ ' HOME ' ] + ' /.cache '
cache_home = cache_home + " /ganarchy "
config_home = os . environ . get ( ' XDG_CONFIG_HOME ' , ' ' )
if not config_home :
config_home = os . environ [ ' HOME ' ] + ' /.config '
config_home = config_home + " /ganarchy "
config_dirs = os . environ . get ( ' XDG_CONFIG_DIRS ' , ' ' )
if not config_dirs :
config_dirs = ' /etc/xdg '
# TODO check if this is correct
config_dirs = [ config_dir + " /ganarchy " for config_dir in config_dirs . split ( ' : ' ) ]
def get_template_loader ( ) :
from jinja2 import DictLoader , FileSystemLoader , ChoiceLoader
return ChoiceLoader ( [
FileSystemLoader ( [ config_home + " /templates " ] + [ config_dir + " /templates " for config_dir in config_dirs ] ) ,
DictLoader ( {
## index.html
' index.html ' : """ <!DOCTYPE html>
< html lang = " en " >
< head >
< meta charset = " utf-8 " / >
< ! - -
GAnarchy - project homepage generator
Copyright ( C ) 2019 Soni L .
This program is free software : you can redistribute it and / or modify
2019-06-16 21:55:56 +02:00
it under the terms of the GNU General Public License as published by
2019-05-28 02:14:09 +02:00
the Free Software Foundation , either version 3 of the License , or
( at your option ) any later version .
This program is distributed in the hope that it will be useful ,
but WITHOUT ANY WARRANTY ; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
2019-06-16 21:55:56 +02:00
GNU General Public License for more details .
2019-05-28 02:14:09 +02:00
2019-06-16 21:55:56 +02:00
You should have received a copy of the GNU General Public License
2019-05-28 02:14:09 +02:00
along with this program . If not , see < https : / / www . gnu . org / licenses / > .
- - >
< title > { { ganarchy . title | e } } < / title >
< meta name = " description " content = " {{ ganarchy.title|e }} " / >
< ! - - if your browser doesn ' t like the following, use a different browser.-->
< script type = " application/javascript " src = " /index.js " > < / script >
< / head >
< body >
< h1 > { { ganarchy . title | e } } < / h1 >
< p > This is { { ganarchy . title | e } } . Currently tracking the following projects : < / p >
< ul >
{ % for project in ganarchy . projects - % }
< li > < a href = " /project/ {{ project.commit|e }} " > { { project . title | e } } < / a > : { { project . description | e } } < / li >
{ % endfor - % }
< / ul >
< p > Powered by < a href = " https://ganarchy.autistic.space/ " > GAnarchy < / a > . AGPLv3 - licensed . < a href = " https://cybre.tech/SoniEx2/ganarchy " > Source Code < / a > . < / p >
< p >
< a href = " {{ ganarchy.base_url|e }} " onclick = " event.preventDefault(); navigator.registerProtocolHandler( ' web+ganarchy ' , this.href + ' ?url= %s ' , ' GAnarchy ' ); " > Register web + ganarchy : URI handler < / a > .
< / p >
< / body >
< / html >
2019-06-20 02:49:27 +02:00
""" ,
## index.toml
' index.toml ' : """ # Generated by GAnarchy
{ % - for project , repos in config . projects . items ( ) % }
[ projects . { { project } } ]
{ % - for repo_url , branches in repos . items ( ) % } { % for branch , options in branches . items ( ) % } { % if options . active % }
" {{ repo_url|tomle}} " . { % if branch % } " {{ branch|tomle}} " { % else % } HEAD { % endif % } = { active = true }
{ % - endif % } { % endfor % }
{ % - endfor % }
{ % endfor - % }
2019-05-28 02:14:09 +02:00
""" ,
## project.html FIXME
' project.html ' : """ <!DOCTYPE html>
2019-04-17 03:58:00 +02:00
< html lang = " en " >
2019-04-20 22:12:39 +02:00
< head >
< meta charset = " utf-8 " / >
< ! - -
GAnarchy - project homepage generator
Copyright ( C ) 2019 Soni L .
This program is free software : you can redistribute it and / or modify
2019-06-16 21:55:56 +02:00
it under the terms of the GNU General Public License as published by
2019-04-20 22:12:39 +02:00
the Free Software Foundation , either version 3 of the License , or
( at your option ) any later version .
This program is distributed in the hope that it will be useful ,
but WITHOUT ANY WARRANTY ; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
2019-06-16 21:55:56 +02:00
GNU General Public License for more details .
2019-04-20 22:12:39 +02:00
2019-06-16 21:55:56 +02:00
You should have received a copy of the GNU General Public License
2019-04-20 22:12:39 +02:00
along with this program . If not , see < https : / / www . gnu . org / licenses / > .
- - >
< title > { { project_title | e } } < / title >
{ % if project_desc % } < meta name = " description " content = " {{ project_desc|e }} " / > { % endif % }
2019-05-19 15:17:04 +02:00
< style type = " text/css " > . branchname { color : #808080; font-style: italic; }</style>
2019-04-20 22:12:39 +02:00
< / head >
< body >
2019-04-21 00:27:50 +02:00
< h1 > { { project_title | e } } < / h1 >
2019-04-23 11:40:23 +02:00
< p > Tracking < span id = " project_commit " > < a href = " web+ganarchy: {{ project_commit }} " > { { project_commit } } < / a > < / span > < / p >
2019-04-21 00:27:50 +02:00
< div id = " project_body " > < p > { { project_body | e | replace ( " \n \n " , " </p><p> " ) } } < / p > < / div >
2019-04-20 22:12:39 +02:00
< ul >
2019-05-19 15:17:04 +02:00
{ % for url , msg , img , branch in repos - % }
< li > < a href = " {{ url|e }} " > { { url | e } } < / a > { % if branch % } < span class = " branchname " > [ { { branch | e } } ] < / span > { % endif % } : { { msg | e } } < / li >
2019-05-28 02:14:09 +02:00
{ % endfor - % }
2019-04-20 22:12:39 +02:00
< / ul >
2019-04-22 23:59:02 +02:00
< p > Powered by < a href = " https://ganarchy.autistic.space/ " > GAnarchy < / a > . AGPLv3 - licensed . < a href = " https://cybre.tech/SoniEx2/ganarchy " > Source Code < / a > . < / p >
< p >
< a href = " / " > Main page < / a > .
2019-04-28 23:50:52 +02:00
< a href = " {{ base_url|e }} " onclick = " event.preventDefault(); navigator.registerProtocolHandler( ' web+ganarchy ' , this.href + ' ?url= %s ' , ' GAnarchy ' ); " > Register web + ganarchy : URI handler < / a > .
2019-04-22 23:59:02 +02:00
< / p >
2019-04-20 22:12:39 +02:00
< / body >
2019-04-17 03:58:00 +02:00
< / html >
2019-05-28 02:14:09 +02:00
""" ,
## history.svg FIXME
' history.svg ' : """ """ ,
} )
] )
2019-04-17 03:58:00 +02:00
2019-06-20 02:49:27 +02:00
tomletrans = str . maketrans ( {
0 : ' \\ u0000 ' , 1 : ' \\ u0001 ' , 2 : ' \\ u0002 ' , 3 : ' \\ u0003 ' , 4 : ' \\ u0004 ' ,
5 : ' \\ u0005 ' , 6 : ' \\ u0006 ' , 7 : ' \\ u0007 ' , 8 : ' \\ b ' , 9 : ' \\ t ' , 10 : ' \\ n ' ,
11 : ' \\ u000B ' , 12 : ' \\ f ' , 13 : ' \\ r ' , 14 : ' \\ u000E ' , 15 : ' \\ u000F ' ,
16 : ' \\ u0010 ' , 17 : ' \\ u0011 ' , 18 : ' \\ u0012 ' , 19 : ' \\ u0013 ' , 20 : ' \\ u0014 ' ,
21 : ' \\ u0015 ' , 22 : ' \\ u0016 ' , 23 : ' \\ u0017 ' , 24 : ' \\ u0018 ' , 25 : ' \\ u0019 ' ,
26 : ' \\ u001A ' , 27 : ' \\ u001B ' , 28 : ' \\ u001C ' , 29 : ' \\ u001D ' , 30 : ' \\ u001E ' ,
31 : ' \\ u001F ' , ' " ' : ' \\ " ' , ' \\ ' : ' \\ \\ '
} )
def tomlescape ( value ) :
return value . translate ( tomletrans )
def get_env ( ) :
env = jinja2 . Environment ( loader = get_template_loader ( ) , autoescape = False )
env . filters [ ' tomlescape ' ] = tomlescape
env . filters [ ' tomle ' ] = env . filters [ ' tomlescape ' ]
return env
2019-04-17 03:58:00 +02:00
@click.group ( )
def ganarchy ( ) :
pass
@ganarchy.command ( )
def initdb ( ) :
2019-05-03 00:14:09 +02:00
""" Initializes the ganarchy database. """
2019-04-17 03:58:00 +02:00
os . makedirs ( data_home , exist_ok = True )
conn = sqlite3 . connect ( data_home + " /ganarchy.db " )
c = conn . cursor ( )
2019-05-19 15:17:04 +02:00
c . execute ( ''' CREATE TABLE " repo_history " ( " entry " INTEGER PRIMARY KEY ASC AUTOINCREMENT, " url " TEXT, " count " INTEGER, " head_commit " TEXT, " branch " TEXT, " project " TEXT) ''' )
c . execute ( ''' CREATE INDEX " repo_history_url_branch_project " ON " repo_history " ( " url " , " branch " , " project " ) ''' )
2019-04-17 03:58:00 +02:00
conn . commit ( )
conn . close ( )
2019-05-03 00:14:09 +02:00
def migrations ( ) :
2019-05-04 01:19:24 +02:00
@ganarchy.group ( )
def migrations ( ) :
""" Modifies the DB to work with a newer/older version.
WARNING : THIS COMMAND CAN BE EXTREMELY DESTRUCTIVE ! """
@migrations.command ( )
@click.argument ( ' migration ' )
def apply ( migration ) :
""" Applies the migration with the given name. """
conn = sqlite3 . connect ( data_home + " /ganarchy.db " )
c = conn . cursor ( )
click . echo ( MIGRATIONS [ migration ] [ 0 ] )
2019-05-19 15:17:04 +02:00
for migration in MIGRATIONS [ migration ] [ 0 ] :
c . execute ( migration )
2019-05-04 01:19:24 +02:00
conn . commit ( )
conn . close ( )
@click.argument ( ' migration ' )
@migrations.command ( )
def revert ( migration ) :
""" Reverts the migration with the given name. """
conn = sqlite3 . connect ( data_home + " /ganarchy.db " )
c = conn . cursor ( )
click . echo ( MIGRATIONS [ migration ] [ 1 ] )
2019-05-19 15:17:04 +02:00
for migration in MIGRATIONS [ migration ] [ 1 ] :
c . execute ( migration )
2019-05-04 01:19:24 +02:00
conn . commit ( )
conn . close ( )
@click.argument ( ' migration ' , required = False )
@migrations.command ( )
def info ( migration ) :
""" Shows information about the migration with the given name. """
if not migration :
# TODO could be improved
click . echo ( MIGRATIONS . keys ( ) )
else :
click . echo ( MIGRATIONS [ migration ] [ 2 ] )
migrations ( )
2019-05-03 00:14:09 +02:00
2019-05-28 02:14:09 +02:00
class GitError ( LookupError ) :
""" Raised when a git operation fails, generally due to a missing commit or branch, or network connection issues. """
pass
2019-06-15 16:03:31 +02:00
class Git :
def __init__ ( self , path ) :
self . path = path
self . base = ( " git " , " -C " , path )
def get_hash ( self , target ) :
try :
return subprocess . check_output ( self . base + ( " show " , target , " -s " , " --format=format: % H " , " -- " ) , stderr = subprocess . DEVNULL ) . decode ( " utf-8 " )
except subprocess . CalledProcessError as e :
raise GitError from e
def get_commit_message ( self , target ) :
try :
return subprocess . check_output ( self . base + ( " show " , target , " -s " , " --format=format: % B " , " -- " ) , stderr = subprocess . DEVNULL ) . decode ( " utf-8 " , " replace " )
except subprocess . CalledProcessError as e :
raise GitError from e
# Currently we only use one git repo, at cache_home
GIT = Git ( cache_home )
2019-05-28 02:14:09 +02:00
class Repo :
2019-06-20 02:49:27 +02:00
def __init__ ( self , dbconn , project_commit , url , branch , head_commit , list_metadata = False ) :
2019-05-28 02:14:09 +02:00
self . url = url
self . branch = branch
2019-06-20 02:49:27 +02:00
self . project_commit = project_commit
self . erroring = False
2019-05-28 02:14:09 +02:00
2019-05-19 15:17:04 +02:00
if not branch :
2019-05-28 02:14:09 +02:00
self . branchname = " gan " + hashlib . sha256 ( url . encode ( " utf-8 " ) ) . hexdigest ( )
self . head = " HEAD "
2019-05-19 15:17:04 +02:00
else :
2019-05-28 02:14:09 +02:00
self . branchname = " gan " + hmac . new ( branch . encode ( " utf-8 " ) , url . encode ( " utf-8 " ) , " sha256 " ) . hexdigest ( )
self . head = " refs/heads/ " + branch
2019-06-15 16:03:31 +02:00
if head_commit :
self . hash = head_commit
else :
2019-06-20 02:49:27 +02:00
try : # FIXME should we even do this?
2019-06-15 16:03:31 +02:00
self . hash = GIT . get_hash ( self . branchname )
except GitError :
2019-06-20 02:49:27 +02:00
self . erroring = True
2019-06-15 16:03:31 +02:00
self . hash = None
2019-05-28 02:14:09 +02:00
self . message = None
if list_metadata :
try :
2019-06-15 16:03:31 +02:00
self . update_metadata ( )
2019-05-28 02:14:09 +02:00
except GitError :
2019-06-20 02:49:27 +02:00
self . erroring = True
2019-05-28 02:14:09 +02:00
pass
2019-06-15 16:03:31 +02:00
def update_metadata ( self ) :
self . message = GIT . get_commit_message ( self . branchname )
2019-05-28 02:14:09 +02:00
2019-06-15 16:03:31 +02:00
def update ( self ) :
"""
Updates the git repo , returning new metadata .
"""
2019-04-17 03:58:00 +02:00
try :
2019-05-28 02:14:09 +02:00
subprocess . check_output ( [ " git " , " -C " , cache_home , " fetch " , " -q " , self . url , " + " + self . head + " : " + self . branchname ] , stderr = subprocess . STDOUT )
2019-04-17 03:58:00 +02:00
except subprocess . CalledProcessError as e :
# This may error for various reasons, but some are important: dead links, etc
click . echo ( e . output , err = True )
2019-06-20 02:49:27 +02:00
self . erroring = True
2019-04-17 03:58:00 +02:00
return None
2019-05-28 02:14:09 +02:00
pre_hash = self . hash
2019-06-15 16:03:31 +02:00
try :
post_hash = GIT . get_hash ( self . branchname )
except GitError as e :
# This should never happen, but maybe there's some edge cases?
2019-06-20 02:49:27 +02:00
# TODO check
self . erroring = True
2019-06-15 16:03:31 +02:00
return None
2019-05-28 02:14:09 +02:00
self . hash = post_hash
2019-04-17 03:58:00 +02:00
if not pre_hash :
pre_hash = post_hash
try :
2019-04-17 04:06:16 +02:00
count = int ( subprocess . check_output ( [ " git " , " -C " , cache_home , " rev-list " , " --count " , pre_hash + " .. " + post_hash , " -- " ] ) . decode ( " utf-8 " ) . strip ( ) )
2019-04-17 03:58:00 +02:00
except subprocess . CalledProcessError :
count = 0 # force-pushed
try :
2019-05-28 02:14:09 +02:00
subprocess . check_call ( [ " git " , " -C " , cache_home , " merge-base " , " --is-ancestor " , self . project_commit , self . branchname ] , stdout = subprocess . DEVNULL , stderr = subprocess . DEVNULL )
2019-06-15 16:03:31 +02:00
self . update_metadata ( )
2019-06-20 02:49:27 +02:00
return count
2019-05-28 02:14:09 +02:00
except ( subprocess . CalledProcessError , GitError ) as e :
click . echo ( e , err = True )
2019-06-20 02:49:27 +02:00
self . erroring = True
2019-04-17 03:58:00 +02:00
return None
2019-05-28 02:14:09 +02:00
class Project :
2019-06-20 02:49:27 +02:00
def __init__ ( self , dbconn , project_commit , list_repos = False ) :
2019-05-28 02:14:09 +02:00
self . commit = project_commit
self . refresh_metadata ( )
2019-06-20 02:49:27 +02:00
self . repos = None
2019-05-28 02:14:09 +02:00
if list_repos :
2019-06-20 02:49:27 +02:00
self . list_repos ( dbconn )
def list_repos ( self , dbconn ) :
repos = [ ]
with dbconn :
for ( e , url , branch , head_commit ) in dbconn . execute ( ''' SELECT " max " ( " e " ), " url " , " branch " , " head_commit " FROM (SELECT " max " ( " T1 " . " entry " ) " e " , " T1 " . " url " , " T1 " . " branch " , " T1 " . " head_commit " FROM " repo_history " " T1 "
WHERE ( SELECT " active " FROM " repos " " T2 " WHERE " url " = " T1 " . " url " AND " branch " IS " T1 " . " branch " AND " project " IS ? 1 )
GROUP BY " T1 " . " url " , " T1 " . " branch "
UNION
SELECT null , " T3 " . " url " , " T3 " . " branch " , null FROM " repos " " T3 " WHERE " active " AND " project " IS ? 1 )
GROUP BY " url " ORDER BY " e " ''' , (self.commit,)):
repos . append ( Repo ( dbconn , self . commit , url , branch , head_commit ) )
self . repos = repos
2019-05-28 02:14:09 +02:00
def refresh_metadata ( self ) :
try :
2019-06-15 16:03:31 +02:00
project = GIT . get_commit_message ( self . commit )
2019-05-28 02:14:09 +02:00
project_title , project_desc = ( lambda x : x . groups ( ) if x is not None else ( ' ' , None ) ) ( re . fullmatch ( ' ^ \\ [Project \\ ] \ s+(.+?)(?: \n \n (.+))?$ ' , project , flags = re . ASCII | re . DOTALL | re . IGNORECASE ) )
2019-06-15 16:03:31 +02:00
if not project_title . strip ( ) : # FIXME
2019-05-28 02:14:09 +02:00
project_title , project_desc = ( " Error parsing project commit " , ) * 2
2019-06-20 02:49:27 +02:00
# if project_desc: # FIXME
# project_desc = project_desc.strip()
2019-05-28 02:14:09 +02:00
self . commit_body = project
self . title = project_title
self . description = project_desc
2019-06-15 16:03:31 +02:00
except GitError :
2019-05-28 02:14:09 +02:00
self . commit_body = None
self . title = None
self . description = None
def update ( self ) :
2019-06-20 02:49:27 +02:00
# TODO? check if working correctly
results = [ ( repo , repo . update ( ) ) for repo in self . repos ]
2019-05-28 02:14:09 +02:00
self . refresh_metadata ( )
2019-06-20 02:49:27 +02:00
return results
2019-05-28 02:14:09 +02:00
class GAnarchy :
2019-06-20 02:49:27 +02:00
def __init__ ( self , dbconn , config , list_projects = False , list_repos = False ) :
base_url = config . base_url
title = config . title
if not base_url :
# FIXME use a more appropriate error type
raise ValueError
if not title :
title = " GAnarchy on " + urlparse ( base_url ) . hostname
self . title = title
self . base_url = base_url
# load config onto DB
c = dbconn . cursor ( )
c . execute ( ''' CREATE TEMPORARY TABLE " repos " ( " url " TEXT PRIMARY KEY, " active " INT, " branch " TEXT, " project " TEXT) ''' )
c . execute ( ''' CREATE UNIQUE INDEX " temp " . " repos_url_branch_project " ON " repos " ( " url " , " branch " , " project " ) ''' )
c . execute ( ''' CREATE INDEX " temp " . " repos_project " ON " repos " ( " project " ) ''' )
c . execute ( ''' CREATE INDEX " temp " . " repos_active " ON " repos " ( " active " ) ''' )
for ( project_commit , repos ) in config . projects . items ( ) :
for ( repo_url , branches ) in repos . items ( ) :
for ( branchname , options ) in branches . items ( ) :
if options [ ' active ' ] : # no need to insert inactive repos since they get ignored anyway
c . execute ( ''' INSERT INTO " repos " VALUES (?, ?, ?, ?) ''' , ( repo_url , 1 , branchname , project_commit ) )
dbconn . commit ( )
2019-05-28 02:14:09 +02:00
if list_projects :
projects = [ ]
with dbconn :
for ( project , ) in dbconn . execute ( ''' SELECT DISTINCT " project " FROM " repos " ''' ) : # FIXME? *maybe* sort by activity in the future
2019-06-20 02:49:27 +02:00
projects . append ( Project ( dbconn , project , list_repos = list_repos ) )
projects . sort ( key = lambda project : project . title ) # sort projects by title
2019-05-28 02:14:09 +02:00
self . projects = projects
else :
self . projects = None
2019-06-20 02:49:27 +02:00
class Config :
def __init__ ( self , toml_file , base = None , remove = True ) :
self . projects = defaultdict ( lambda : defaultdict ( lambda : defaultdict ( lambda : defaultdict ( dict ) ) ) )
config_data = qtoml . load ( toml_file )
self . title = config_data . get ( ' title ' , ' ' )
self . base_url = config_data . get ( ' base_url ' , ' ' )
# TODO blocked domains (but only read them from config_data if remove is True)
self . blocked_domains = [ ]
self . blocked_domain_suffixes = [ ]
self . blocked_domains . sort ( )
self . blocked_domain_suffixes . sort ( key = lambda x : x [ : : - 1 ] )
# FIXME remove duplicates and process invalid entries
self . blocked_domains = tuple ( self . blocked_domains )
self . blocked_domain_suffixes = tuple ( self . blocked_domain_suffixes ) # MUST be tuple
# TODO re.compile("(^" + "|^".join(map(re.escape, domains)) + "|" + "|".join(map(re.escape, suffixes) + ")$")
if base :
self . _update_projects ( base . projects , sanitize = False ) # already sanitized
projects = config_data . get ( ' projects ' , { } )
self . _update_projects ( projects , remove = remove )
def _update_projects ( self , projects , remove , sanitize = True ) :
for ( project_commit , repos ) in projects . items ( ) :
if sanitize and not isinstance ( repos , dict ) :
# TODO emit warnings?
continue
if sanitize and not re . fullmatch ( " [0-9a-fA-F] {40} |[0-9a-fA-F] {64} " , project_commit ) : # future-proofing: sha256 support
# TODO emit warnings?
continue
project = self . projects [ project_commit ]
for ( repo_url , branches ) in repos . items ( ) :
if sanitize and not isinstance ( branches , dict ) :
# TODO emit warnings?
continue
try :
u = urlparse ( repo_url )
if not u :
raise ValueError
getattr ( u , ' port ' ) # raises ValueError if port is invalid
if u . scheme in ( ' file ' , ' ' ) :
raise ValueError
if ( u . hostname in self . blocked_domains ) or ( u . hostname . endswith ( self . blocked_domain_suffixes ) ) :
raise ValueError
except ValueError :
if sanitize :
# TODO emit warnings?
continue
else :
raise
repo = project [ repo_url ]
for ( branchname , options ) in branches . items ( ) :
if sanitize and not isinstance ( options , dict ) :
# TODO emit warnings?
continue
if branchname == " HEAD " :
if sanitize :
# feels weird, but generally makes things easier
# DO NOT emit warnings here. this is deliberate.
branchname = None
else :
raise ValueError
branch = repo [ branchname ]
active = options . get ( ' active ' , False )
if active not in ( True , False ) :
if sanitize :
# TODO emit warnings?
continue
else :
raise ValueError
## | remove | branch.active | options.active | result |
## | x | false | false | false |
## | x | false | true | true |
## | x | true | true | true |
## | false | true | false | true |
## | true | true | false | false |
branch [ ' active ' ] = branch . get ( ' active ' , False ) or active
if remove and not active :
branch [ ' active ' ] = False
@ganarchy.command ( )
@click.option ( ' --skip-errors/--no-skip-errors ' , default = False )
@click.argument ( ' files ' , type = click . File ( ' r ' , encoding = ' utf-8 ' ) , nargs = - 1 )
def merge_configs ( skip_errors , files ) :
""" Merges config files. """
config = None
for f in files :
try :
f . reconfigure ( newline = ' ' )
config = Config ( f , config , remove = False )
except ( UnicodeDecodeError , qtoml . decoder . TOMLDecodeError ) :
if not skip_errors :
raise
if config :
env = get_env ( )
template = env . get_template ( ' index.toml ' )
click . echo ( template . render ( config = config ) )
2019-05-28 02:14:09 +02:00
@ganarchy.command ( )
@click.argument ( ' project ' , required = False )
def cron_target ( project ) :
""" Runs ganarchy as a cron target. """
2019-06-20 02:49:27 +02:00
conf = None
# reverse order is intentional
for d in reversed ( config_dirs ) :
try :
conf = Config ( open ( d + " /config.toml " , ' r ' , encoding = ' utf-8 ' , newline = ' ' ) , conf )
except ( OSError , UnicodeDecodeError , qtoml . decoder . TOMLDecodeError ) :
pass
with open ( config_home + " /config.toml " , ' r ' , encoding = ' utf-8 ' , newline = ' ' ) as f :
conf = Config ( f , conf )
env = get_env ( )
if project == " config " :
# render the config
# doesn't have access to a GAnarchy object. this is deliberate.
template = env . get_template ( ' index.toml ' )
click . echo ( template . render ( config = conf ) )
return
2019-05-28 02:14:09 +02:00
# make sure the cache dir exists
2019-04-17 03:58:00 +02:00
os . makedirs ( cache_home , exist_ok = True )
2019-05-28 02:14:09 +02:00
# make sure it is a git repo
2019-04-17 03:58:00 +02:00
subprocess . call ( [ " git " , " -C " , cache_home , " init " , " -q " ] )
conn = sqlite3 . connect ( data_home + " /ganarchy.db " )
2019-06-20 02:49:27 +02:00
instance = GAnarchy ( conn , conf , list_projects = project in [ " index " , " config " ] )
2019-05-28 02:14:09 +02:00
if project == " index " :
# render the index
template = env . get_template ( ' index.html ' )
click . echo ( template . render ( ganarchy = instance ) )
return
2019-06-20 02:49:27 +02:00
if not instance . base_url or not project :
2019-04-22 23:59:02 +02:00
click . echo ( " No base URL or project commit specified " , err = True )
return
2019-04-17 03:58:00 +02:00
entries = [ ]
generate_html = [ ]
2019-05-28 02:14:09 +02:00
c = conn . cursor ( )
2019-06-20 02:49:27 +02:00
p = Project ( conn , project , list_repos = True )
results = p . update ( )
for ( repo , count ) in results :
if count is not None :
entries . append ( ( repo . url , count , repo . hash , repo . branch , project ) )
generate_html . append ( ( repo . url , repo . message , count , repo . branch ) )
2019-04-19 21:42:59 +02:00
# sort stuff twice because reasons
entries . sort ( key = lambda x : x [ 1 ] , reverse = True )
generate_html . sort ( key = lambda x : x [ 2 ] , reverse = True )
2019-05-19 15:17:04 +02:00
c . executemany ( ''' INSERT INTO " repo_history " ( " url " , " count " , " head_commit " , " branch " , " project " ) VALUES (?, ?, ?, ?, ?) ''' , entries )
2019-04-17 03:58:00 +02:00
conn . commit ( )
html_entries = [ ]
2019-05-19 15:17:04 +02:00
for ( url , msg , count , branch ) in generate_html :
history = c . execute ( ''' SELECT " count " FROM " repo_history " WHERE " url " = ? AND " branch " IS ? AND " project " IS ? ORDER BY " entry " ASC ''' , ( url , branch , project ) ) . fetchall ( )
2019-04-17 03:58:00 +02:00
# TODO process history into SVG
2019-05-19 15:17:04 +02:00
html_entries . append ( ( url , msg , " " , branch ) )
2019-05-28 02:14:09 +02:00
template = env . get_template ( ' project.html ' )
click . echo ( template . render ( project_title = p . title ,
project_desc = p . description ,
project_body = p . commit_body ,
project_commit = p . commit ,
2019-04-22 23:59:02 +02:00
repos = html_entries ,
2019-06-20 02:49:27 +02:00
base_url = instance . base_url ,
2019-05-28 02:14:09 +02:00
# I don't think this thing supports deprecating the above?
project = p ,
ganarchy = instance ) )
2019-04-17 03:58:00 +02:00
if __name__ == " __main__ " :
ganarchy ( )