From d73c7e44d5fc0eb5f108778ec6fdfd67fe23b3cd Mon Sep 17 00:00:00 2001 From: magmaus3 Date: Sat, 17 Dec 2022 19:10:58 +0100 Subject: [PATCH] Add thumbnail support! --- customiwmserver/__main__.py | 14 +++++- customiwmserver/main.py | 14 +++--- customiwmserver/object_storage.py | 29 +++++++++++ docker-compose.yml | 14 ++++++ poetry.lock | 82 ++++++++++++++++++++++++++++++- pyproject.toml | 2 + 6 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 customiwmserver/object_storage.py diff --git a/customiwmserver/__main__.py b/customiwmserver/__main__.py index 0daaea9..438c1f4 100644 --- a/customiwmserver/__main__.py +++ b/customiwmserver/__main__.py @@ -1,4 +1,16 @@ import uvicorn +import signal + +config = uvicorn.Config("customiwmserver.main:app", host="0.0.0.0", port=8001, reload=True) +server = uvicorn.Server(config) + +def exit_now(signum, frame): + global server + print("STOPPING") + server.handle_exit(signum, frame) + +signal.signal(signal.SIGINT, exit_now) +signal.signal(signal.SIGTERM, exit_now) if __name__ == "__main__": - uvicorn.run("customiwmserver.main:app", host="0.0.0.0", port=8001, reload=True) + server.run() diff --git a/customiwmserver/main.py b/customiwmserver/main.py index b7a5582..ed57ade 100644 --- a/customiwmserver/main.py +++ b/customiwmserver/main.py @@ -11,7 +11,7 @@ from . import data_types as types from . import database as db from . import hook_system from . import hooks - +from . import object_storage import pymongo app = FastAPI() @@ -28,8 +28,7 @@ async def http_exception_handler(request, exc): return PlainTextResponse( f"[{exc.status_code}] {str(exc.detail)}", status_code=exc.status_code ) - - + ## Users @@ -323,6 +322,7 @@ async def upload_map( userData = types.User(**authcheck[1]) ID = len(list(db.maps_collection.find({}))) + 1 MapCode = db.id_to_mapcode(ID) + await object_storage.upload_thumbnail(file, ID) db.maps_collection.insert_one( { **types.Map( @@ -369,6 +369,7 @@ async def upload_map( ], ).dict() } + ) return {"MapCode": MapCode} # raise HTTPException(501) @@ -587,9 +588,10 @@ async def featuredlist(): async def followcheck(): """Check, if creators that the user follows uploaded new levels.""" # FIXME: Stub - raise HTTPException(404, detail="Not available due to follows not being implemented.") - + return 0 def start(): """Launched with `poetry run start` at root level""" - uvicorn.run("customiwmserver.main:app", host="0.0.0.0", port=8001, reload=True) + from . import __main__ + __main__.server.run() + # uvicorn.run("customiwmserver.main:app", host="0.0.0.0", port=8001, reload=True) diff --git a/customiwmserver/object_storage.py b/customiwmserver/object_storage.py new file mode 100644 index 0000000..62c98cb --- /dev/null +++ b/customiwmserver/object_storage.py @@ -0,0 +1,29 @@ +from minio import Minio +from os import getenv +import json +import io + +print("---") +print(getenv("S3_ENDPOINT"), + getenv("S3_ACCESS_KEY"), + getenv("S3_SECRET_KEY"), + json.loads(getenv("S3_IS_SECURE", "false")), sep="\n" +) +print("---") + +client = Minio( + getenv("S3_ENDPOINT"), + access_key=getenv("S3_ACCESS_KEY"), + secret_key=getenv("S3_SECRET_KEY"), + secure=json.loads(getenv("S3_IS_SECURE", "false")), +) + +ThumbsBucket = "iwm-user-thumbnails" + +if not client.bucket_exists(ThumbsBucket): + client.make_bucket(ThumbsBucket) + +async def upload_thumbnail(data, levelID): + # file = io.BytesIO(await data.read()) + client.put_object(ThumbsBucket, f"{levelID}.png", data.file, length=-1, part_size=10*1024*1024) + diff --git a/docker-compose.yml b/docker-compose.yml index e3afd2f..32213d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,11 @@ services: - "8001:8001" volumes: - .:/app + environment: + - S3_ENDPOINT=minio:9000 + - S3_IS_SECURE=false + - S3_ACCESS_KEY=REPLACE_ME + - S3_SECRET_KEY=REPLACE_ME depends_on: - mongo @@ -30,3 +35,12 @@ services: ME_CONFIG_MONGODB_URL: mongodb://root:catboys@mongo:27017/ depends_on: - mongo + + minio: + image: minio/minio + ports: + - 9000:9000 + - 9001:9001 + volumes: + - /data + command: "server /data --console-address ':9001'" diff --git a/poetry.lock b/poetry.lock index 9f80fa2..e585fe4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -37,6 +37,14 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +[[package]] +name = "certifi" +version = "2022.9.24" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "click" version = "8.1.3" @@ -106,6 +114,18 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "minio" +version = "7.1.12" +description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +certifi = "*" +urllib3 = "*" + [[package]] name = "more-itertools" version = "9.0.0" @@ -260,6 +280,19 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "urllib3" +version = "1.26.13" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "uvicorn" version = "0.19.0" @@ -275,6 +308,17 @@ h11 = ">=0.8" [package.extras] standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.0)"] +[[package]] +name = "watchfiles" +version = "0.18.1" +description = "Simple, modern and high performance file watching and code reload in python." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +anyio = ">=3.0.0" + [[package]] name = "wcwidth" version = "0.2.5" @@ -286,18 +330,24 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "6b0e9b1efc1d8035937fb8b8bc374bfae12d6e7658f8273b63450174b57f0a24" +content-hash = "7f72744b65ebfa4a6d4b807d750f36e1cef4ed2c52d2527a4c80c2b9fb3f950f" [metadata.files] anyio = [ {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, ] -atomicwrites = [] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] attrs = [ {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, ] +certifi = [ + {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, + {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, +] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, @@ -322,6 +372,10 @@ idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] +minio = [ + {file = "minio-7.1.12-py3-none-any.whl", hash = "sha256:c8ab8646f93d47b9aefbf4db76aaba5ac54c87454b922a3d6c1423aed050aad5"}, + {file = "minio-7.1.12.tar.gz", hash = "sha256:63111fedf67e07c5a4c8948b3a4e5ecbb372b522ea562bfa4d484194ec6a2b99"}, +] more-itertools = [ {file = "more-itertools-9.0.0.tar.gz", hash = "sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab"}, {file = "more_itertools-9.0.0-py3-none-any.whl", hash = "sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41"}, @@ -479,10 +533,34 @@ typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] +urllib3 = [ + {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, + {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, +] uvicorn = [ {file = "uvicorn-0.19.0-py3-none-any.whl", hash = "sha256:cc277f7e73435748e69e075a721841f7c4a95dba06d12a72fe9874acced16f6f"}, {file = "uvicorn-0.19.0.tar.gz", hash = "sha256:cf538f3018536edb1f4a826311137ab4944ed741d52aeb98846f52215de57f25"}, ] +watchfiles = [ + {file = "watchfiles-0.18.1-cp37-abi3-macosx_10_7_x86_64.whl", hash = "sha256:9891d3c94272108bcecf5597a592e61105279def1313521e637f2d5acbe08bc9"}, + {file = "watchfiles-0.18.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:7102342d60207fa635e24c02a51c6628bf0472e5fef067f78a612386840407fc"}, + {file = "watchfiles-0.18.1-cp37-abi3-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:00ea0081eca5e8e695cffbc3a726bb90da77f4e3f78ce29b86f0d95db4e70ef7"}, + {file = "watchfiles-0.18.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8e6db99e49cd7125d8a4c9d33c0735eea7b75a942c6ad68b75be3e91c242fb"}, + {file = "watchfiles-0.18.1-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bc7c726855f04f22ac79131b51bf0c9f728cb2117419ed830a43828b2c4a5fcb"}, + {file = "watchfiles-0.18.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbaff354d12235002e62d9d3fa8bcf326a8490c1179aa5c17195a300a9e5952f"}, + {file = "watchfiles-0.18.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:888db233e06907c555eccd10da99b9cd5ed45deca47e41766954292dc9f7b198"}, + {file = "watchfiles-0.18.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:dde79930d1b28f15994ad6613aa2865fc7a403d2bb14585a8714a53233b15717"}, + {file = "watchfiles-0.18.1-cp37-abi3-win32.whl", hash = "sha256:e2b2bdd26bf8d6ed90763e6020b475f7634f919dbd1730ea1b6f8cb88e21de5d"}, + {file = "watchfiles-0.18.1-cp37-abi3-win_amd64.whl", hash = "sha256:c541e0f2c3e95e83e4f84561c893284ba984e9d0025352057396d96dceb09f44"}, + {file = "watchfiles-0.18.1-cp37-abi3-win_arm64.whl", hash = "sha256:9a26272ef3e930330fc0c2c148cc29706cc2c40d25760c7ccea8d768a8feef8b"}, + {file = "watchfiles-0.18.1-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:9fb12a5e2b42e0b53769455ff93546e6bc9ab14007fbd436978d827a95ca5bd1"}, + {file = "watchfiles-0.18.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:548d6b42303d40264118178053c78820533b683b20dfbb254a8706ca48467357"}, + {file = "watchfiles-0.18.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0d8fdfebc50ac7569358f5c75f2b98bb473befccf9498cf23b3e39993bb45a"}, + {file = "watchfiles-0.18.1-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0f9a22fff1745e2bb930b1e971c4c5b67ea3b38ae17a6adb9019371f80961219"}, + {file = "watchfiles-0.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b02e7fa03cd4059dd61ff0600080a5a9e7a893a85cb8e5178943533656eec65e"}, + {file = "watchfiles-0.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a868ce2c7565137f852bd4c863a164dc81306cae7378dbdbe4e2aca51ddb8857"}, + {file = "watchfiles-0.18.1.tar.gz", hash = "sha256:4ec0134a5e31797eb3c6c624dbe9354f2a8ee9c720e0b46fc5b7bab472b7c6d4"}, +] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, diff --git a/pyproject.toml b/pyproject.toml index 45db4d1..8ff99aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ fastapi = "^0.86.0" pymongo = "^4.3.2" uvicorn = "^0.19.0" python-multipart = "^0.0.5" +minio = "^7.1.12" +watchfiles = "^0.18.1" [tool.poetry.dev-dependencies] pytest = "^5.2"