Compare commits
7 Commits
Author | SHA1 | Date |
---|---|---|
Rampoina | d6f387bcc4 | |
Rampoina | c329a5a428 | |
Rampoina | 6f926a321c | |
Rampoina | 71ca47846d | |
Rampoina | 7180d96567 | |
Rampoina | b3826ad1d7 | |
Armen | e4dcb89d2e |
|
@ -9,8 +9,13 @@ RUN mkdir /var/cdb
|
|||
RUN chown -R cdb:cdb /var/cdb
|
||||
|
||||
COPY requirements.lock.txt requirements.lock.txt
|
||||
RUN apt update
|
||||
RUN apt install -y vim
|
||||
RUN apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-3.0
|
||||
RUN apt install -y libappstream-glib-dev
|
||||
RUN pip install -r requirements.lock.txt
|
||||
RUN pip install gunicorn
|
||||
RUN pip3 install pycairo PyGObject
|
||||
|
||||
COPY utils utils
|
||||
COPY config.cfg config.cfg
|
||||
|
|
|
@ -0,0 +1,68 @@
|
|||
version: '3'
|
||||
|
||||
|
||||
services:
|
||||
db:
|
||||
image: "postgres:14.1"
|
||||
volumes:
|
||||
- "./data/db:/var/lib/postgresql/data"
|
||||
environment: &env
|
||||
- POSTGRES_USER=contentdb
|
||||
- POSTGRES_PASSWORD=password
|
||||
- POSTGRES_DB=contentdb
|
||||
- FLASK_DEBUG=1
|
||||
- FLASK_CONFIG=../config.cfg
|
||||
adminer:
|
||||
image: adminer
|
||||
restart: always
|
||||
environment:
|
||||
- ADMINER_DEFAULT_SERVER=db
|
||||
- ADMINER_DESIGN=pepa-linha
|
||||
ports:
|
||||
- 8081:8080
|
||||
|
||||
redis:
|
||||
image: 'redis:6.2-alpine'
|
||||
command: redis-server
|
||||
volumes:
|
||||
- './data/redis:/data'
|
||||
|
||||
app:
|
||||
build: .
|
||||
container_name: contentdb_app_1
|
||||
command: ./utils/entrypoint.sh
|
||||
environment: *env
|
||||
ports:
|
||||
- 5123:5123
|
||||
volumes:
|
||||
- "./data/uploads:/var/cdb/uploads"
|
||||
- "./app:/source/app"
|
||||
- "./migrations:/source/migrations"
|
||||
depends_on:
|
||||
- db
|
||||
- redis
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: celery -A app.tasks.celery worker --concurrency 1
|
||||
environment: *env
|
||||
volumes:
|
||||
- "./data/uploads:/var/cdb/uploads"
|
||||
- "./app:/home/cdb/app"
|
||||
depends_on:
|
||||
- redis
|
||||
|
||||
beat:
|
||||
build: .
|
||||
command: celery -A app.tasks.celery beat
|
||||
environment: *env
|
||||
depends_on:
|
||||
- redis
|
||||
|
||||
exporter:
|
||||
image: ovalmoney/celery-exporter
|
||||
environment: *env
|
||||
ports:
|
||||
- 5125:9540
|
||||
depends_on:
|
||||
- redis
|
|
@ -1,11 +1,17 @@
|
|||
version: '3'
|
||||
|
||||
|
||||
services:
|
||||
db:
|
||||
image: "postgres:14.1"
|
||||
volumes:
|
||||
- "./data/db:/var/lib/postgresql/data"
|
||||
env_file:
|
||||
- config.env
|
||||
environment: &env
|
||||
- POSTGRES_USER=contentdb
|
||||
- POSTGRES_PASSWORD=password
|
||||
- POSTGRES_DB=contentdb
|
||||
- FLASK_DEBUG=0
|
||||
- FLASK_CONFIG=../config.cfg
|
||||
|
||||
redis:
|
||||
image: 'redis:6.2-alpine'
|
||||
|
@ -15,9 +21,9 @@ services:
|
|||
|
||||
app:
|
||||
build: .
|
||||
container_name: contentdb_app_1
|
||||
command: ./utils/entrypoint.sh
|
||||
env_file:
|
||||
- config.env
|
||||
environment: *env
|
||||
ports:
|
||||
- 5123:5123
|
||||
volumes:
|
||||
|
@ -31,10 +37,7 @@ services:
|
|||
worker:
|
||||
build: .
|
||||
command: celery -A app.tasks.celery worker --concurrency 1
|
||||
env_file:
|
||||
- config.env
|
||||
environment:
|
||||
- FLASK_CONFIG=../config.cfg
|
||||
environment: *env
|
||||
volumes:
|
||||
- "./data/uploads:/var/cdb/uploads"
|
||||
- "./app:/home/cdb/app"
|
||||
|
@ -44,17 +47,13 @@ services:
|
|||
beat:
|
||||
build: .
|
||||
command: celery -A app.tasks.celery beat
|
||||
env_file:
|
||||
- config.env
|
||||
environment:
|
||||
- FLASK_CONFIG=../config.cfg
|
||||
environment: *env
|
||||
depends_on:
|
||||
- redis
|
||||
|
||||
exporter:
|
||||
image: ovalmoney/celery-exporter
|
||||
env_file:
|
||||
- config.env
|
||||
environment: *env
|
||||
ports:
|
||||
- 5125:9540
|
||||
depends_on:
|
||||
|
|
|
@ -0,0 +1,158 @@
|
|||
import gi
|
||||
import PIL
|
||||
import requests
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
from gi.repository import Gio
|
||||
gi.require_version('AppStreamGlib', '1.0')
|
||||
from gi.repository import AppStreamGlib
|
||||
from lists import alwaysAccept, alwaysDeny, badLicenses, badCategories, nonFreeAssets, nonFreeNetworkServices
|
||||
import itertools
|
||||
import argparse
|
||||
|
||||
#Workaround to get the urls because app.get_urls() doesn't work :|
|
||||
def get_urls(app):
|
||||
kinds = [AppStreamGlib.UrlKind(kind) for kind in range(11)]
|
||||
urls = [(app.get_url_item(kind),kind.value_nick) for kind in kinds]
|
||||
return list(filter(lambda a: a[0] is not None, urls))
|
||||
|
||||
def acceptedGame(app):
|
||||
#return 'Game' in app.get_categories()
|
||||
if app.get_id() in alwaysAccept:
|
||||
return True
|
||||
if app.get_id() in alwaysDeny:
|
||||
return False
|
||||
|
||||
return app.get_project_license() and \
|
||||
not [x for x in badLicenses if x in app.get_project_license()] and \
|
||||
'Game' in app.get_categories() and \
|
||||
not [x for x in badCategories if x in app.get_categories()]
|
||||
|
||||
def getScreenshots(app):
|
||||
return [images.get_source() for images in app.get_screenshots()]
|
||||
|
||||
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
|
||||
parentdir = os.path.dirname(currentdir)
|
||||
sys.path.insert(0,parentdir)
|
||||
|
||||
from app.models import *
|
||||
from app.utils import make_flask_login_password
|
||||
from app.utils.image import get_image_size
|
||||
from app.utils import randomString
|
||||
|
||||
parser = argparse.ArgumentParser(description='Generate HTML files from an Appimage and mako templates.')
|
||||
parser.add_argument("-p", '--appstream_path', help='specify the path of the appstream file')
|
||||
parser.add_argument("-i", '--icons_path', help='specify the path of the icons')
|
||||
parser.add_argument("-m", '--template_path', help='specify the path of the template')
|
||||
args = parser.parse_args()
|
||||
|
||||
store = AppStreamGlib.Store()
|
||||
file = Gio.File.new_for_path(args.appstream_path)
|
||||
AppStreamGlib.Store.from_file(store, file, args.icons_path, None)
|
||||
apps = list(filter(acceptedGame, store.get_apps()))
|
||||
session=db.session
|
||||
licenses = { x.name : x for x in License.query.all() }
|
||||
tags = { x.name : x for x in Tag.query.all() }
|
||||
admin_user = User.query.filter_by(username="appstream").first()
|
||||
|
||||
|
||||
for ss in PackageScreenshot.query.all():
|
||||
if ss.package.cover_image == ss:
|
||||
ss.package.cover_image = None
|
||||
session.commit()
|
||||
session.delete(ss)
|
||||
session.commit()
|
||||
|
||||
for game in Package.query.all():
|
||||
session.delete(game)
|
||||
session.commit()
|
||||
|
||||
if not admin_user:
|
||||
admin_user = User("appstream")
|
||||
admin_user.is_active = True
|
||||
admin_user.password = make_flask_login_password("appstream")
|
||||
admin_user.github_username = "appstream"
|
||||
admin_user.forums_username = "appstream"
|
||||
admin_user.rank = UserRank.ADMIN
|
||||
session.add(admin_user)
|
||||
|
||||
|
||||
for app in apps:
|
||||
screenshots = getScreenshots(app)
|
||||
urls = get_urls(app)
|
||||
filename = app.get_name().replace(':', '').replace('/','') + ".html"
|
||||
print("APPLICATION: ",app.get_name())
|
||||
game1 = Package()
|
||||
game1.state = PackageState.APPROVED
|
||||
game1.name = app.get_id().replace('.','_').replace('-','_')
|
||||
game1.title = app.get_name()
|
||||
game1.type = PackageType.GAME
|
||||
license = "Uknown" if app.get_project_license() is None else app.get_project_license().split("AND")[0].split("and")[0]
|
||||
if license not in licenses:
|
||||
row = License(license)
|
||||
licenses[row.name] = row
|
||||
session.add(row)
|
||||
session.commit()
|
||||
for category in app.get_categories():
|
||||
if category.lower() not in tags:
|
||||
row = Tag(category)
|
||||
tags[row.name] = row
|
||||
print("adding tag: ", row.name)
|
||||
session.add(row)
|
||||
game1.tags.append(tags[category.lower()])
|
||||
game1.license = licenses[license]
|
||||
game1.media_license = licenses["MIT"]
|
||||
game1.author = admin_user
|
||||
|
||||
|
||||
for url,t in urls:
|
||||
if t == "bugtracker":
|
||||
game1.issueTracker = url
|
||||
elif t == "homepage":
|
||||
game1.repo = url
|
||||
|
||||
game1.forums = 12835
|
||||
game1.short_desc = "" or app.get_comment()
|
||||
game1.desc = app.get_description()
|
||||
session.add(game1)
|
||||
|
||||
for screenshot in screenshots:
|
||||
counter = 1
|
||||
url = screenshot.get_url()
|
||||
try:
|
||||
r = requests.get(url,timeout=10)
|
||||
r.raise_for_status()
|
||||
filename = randomString(10) + "." + "png"
|
||||
filepath = os.path.join("/var/cdb/uploads", filename)
|
||||
print("Screenshot url: ", url)
|
||||
with open(filepath,"wb") as f:
|
||||
f.write(r.content)
|
||||
|
||||
width, height = get_image_size(filepath)
|
||||
|
||||
if (width is not None) and (height is not None):
|
||||
ss = PackageScreenshot()
|
||||
ss.package = game1
|
||||
ss.title = "Untitled"
|
||||
ss.url = "/uploads/" + filename
|
||||
ss.width = width
|
||||
ss.height = height
|
||||
ss.approved = True
|
||||
ss.order = counter
|
||||
session.add(ss)
|
||||
session.commit()
|
||||
game1.cover_image = ss
|
||||
session.commit()
|
||||
counter += 1
|
||||
except requests.exceptions.HTTPError as err:
|
||||
print("HTTP error downloading the screenshot ", err)
|
||||
except requests.exceptions.ConnectionError as err:
|
||||
print("HTTP error downloading the screenshot ", err)
|
||||
except requests.exceptions.ReadTimeout as err:
|
||||
print("Screenshot timeout ", err)
|
||||
except PIL.UnidentifiedImageError as err:
|
||||
print("Corrupt image ", err)
|
||||
|
||||
session.commit()
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
badLicenses = [
|
||||
'LicenseRef-proprietary',
|
||||
'LicenseRef-Proprietary',
|
||||
'proprietary',
|
||||
'Proprietary',
|
||||
'CC-BY-NC-SA-3.0',
|
||||
'CC-BY-NC-ND-3.0'
|
||||
]
|
||||
|
||||
badCategories = [
|
||||
'Emulator',
|
||||
'PackageManager',
|
||||
'System',
|
||||
'Utility'
|
||||
]
|
||||
|
||||
nonFreeAssets = [
|
||||
'jp.yvt.OpenSpades',
|
||||
'net.openra.OpenRA',
|
||||
'org.openmw.OpenMW',
|
||||
'org.zdoom.GZDoom',
|
||||
'io.github.ezQuake',
|
||||
'com.etlegacy.ETLegacy',
|
||||
'com.github.iortcw.iortcw',
|
||||
'org.yamagi.YamagiQ2',
|
||||
'org.dhewm3.Dhewm3',
|
||||
'com.github.bvschaik.julius',
|
||||
'io.openrct2.OpenRCT2',
|
||||
'com.github.skullernet.q2pro',
|
||||
'org.raceintospace.Raceintospace',
|
||||
'org.srb2.SRB2',
|
||||
'org.srb2.SRB2Kart',
|
||||
'io.sourceforge.clonekeenplus',
|
||||
'io.github.fabiangreffrath.Doom',
|
||||
'net.dengine.Doomsday',
|
||||
'com.github.keriew.augustus',
|
||||
'io.github.yairm210.unciv',
|
||||
'com.corsixth.corsixth'
|
||||
]
|
||||
|
||||
nonFreeNetworkServices = [
|
||||
'io.github.yairm210.unciv'
|
||||
]
|
||||
|
||||
alwaysAccept = [
|
||||
'org.freecol.FreeCol',
|
||||
'org.freeciv.Freeciv',
|
||||
'io.github.EndlessSky.endless-sky',
|
||||
'org.frozen_bubble.frozen-bubble',
|
||||
'org.kde.ksudoku',
|
||||
]
|
||||
|
||||
alwaysDeny = [
|
||||
'com.moonlight_stream.Moonlight',
|
||||
'org.gnome.Games',
|
||||
'org.ppsspp.PPSSPP',
|
||||
'org.scummvm.ScummVM',
|
||||
'org.pegasus_frontend.Pegasus',
|
||||
'com.gitlab.coringao.cavestory-nx',
|
||||
'org.sauerbraten.Sauerbraten',
|
||||
'net.runelite.RuneLite',
|
||||
'com.zandronum.Zandronum',
|
||||
'io.mrarm.mcpelauncher',
|
||||
'org.unitystation.StationHub',
|
||||
'org.firestormviewer.FirestormViewer',
|
||||
'com.eduke32.EDuke32',
|
||||
'io.github.hmlendea.geforcenow-electron',
|
||||
'io.gdevs.GDLauncher',
|
||||
'io.github.sharkwouter.Minigalaxy',
|
||||
'com.katawa_shoujo.KatawaShoujo',
|
||||
're.chiaki.Chiaki'
|
||||
]
|
Loading…
Reference in New Issue