V1.1 mongodb

.gitignore

@@ -10,4 +10,6 @@ data.json
 *.pptx
 /backend/output
 *.pyc
-*.log
+*.log
+mongodb/
+db-backup/

README.md

@@ -1,5 +1,10 @@
+
 # knowlix
 
+
 Microservice to autmatically create onboarding slides
 
 ![LIX Builder](https://github.com/vg-leanix/pptx-tool/blob/main/Thumbnail.png)
+
+## Architecture 
+![Architecture](https://github.com/vg-leanix/knowlix/blob/v1.1_mongodb/knowlix%20architecture.png)

backend/.gitignore

@@ -0,0 +1 @@
+env/

backend/Dockerfile

@@ -2,10 +2,11 @@ FROM python:3.8.1
 
 WORKDIR /usr/app
 
+
 COPY req.txt ./
 COPY api.py core.py main.py master.pptx req.txt server.py ./
 
-# RUN mkdir output 
+RUN mkdir output  
 
 RUN pip install --upgrade pip
 RUN pip install -r req.txt --no-cache-dir

backend/api.py

@@ -9,31 +9,36 @@
 from pptx import Presentation
 from server import celery
 import json
+import uuid
+from datetime import datetime
+from pymongo import MongoClient
 
 
 file_path = "master.pptx"
-pres= Presentation(file_path)
+pres = Presentation(file_path)
+MONGODB = os.getenv("MONGODB")
+client = MongoClient(MONGODB)
+db = client["taskdb"]["ta"]
 
-tags_metadata= [
+tags_metadata = [
     {
-    "name": "powerpoint",
-    "description": "handling powerpoint"
+        "name": "powerpoint",
+        "description": "handling powerpoint"
     },
     {
-    "name": "job management",
-    "description": "managing celery tasks"
+        "name": "job management",
+        "description": "managing celery tasks"
     },
-    
+
 ]
 
 app = FastAPI(
-    title= "SurfBoard",
-    description= "API Hub for the LeanIX Onboarding Deck",
-    version= "1.0.0",
+    title="Knowlix",
+    description="API Hub for the LeanIX Onboarding Deck",
+    version="1.0.0",
     openapi_tags=tags_metadata)
 
 
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -43,42 +48,91 @@
     expose_headers=[]
 )
 
+
 class PPTX(BaseModel):
     sections: List[str]
-
-
-@app.get("/v1/sections", tags = ["powerpoint"])
+
+
+class Download(BaseModel):
+    taskID: str
+
+
+@app.get("/v1/sections", tags=["powerpoint"])
 async def provide_sections():
 
     sections = get_sections(pres)
 
-    if (not sections) or (len(sections)==0):
-        raise HTTPException(status_code=404, detail="No Sections in Master pptx")
+    if (not sections) or (len(sections) == 0):
+        raise HTTPException(
+            status_code=404, detail="No Sections in Master pptx")
 
-    return JSONResponse(sections,status_code=200)
+    return JSONResponse(sections, status_code=200)
 
 
-@app.post("/v1/pptxjob", tags = ["job management"])
+@app.post("/v1/pptxjob", tags=["job management"])
 async def deliver_pptx(pptx: PPTX):
     task_name = "pptx"
     sections = pptx.sections
-    kwargs ={
-        'sections':sections,
-        'downloadStatus': 'ready'
-        }
-
+    no_sections = len(sections)
+    sections_available = True
+    exists_already = False
+    status = None
+    custom_id = str(uuid.uuid4().hex)
+    kwargs = {
+        'sections': sections,
+        'customID': custom_id,
+        'downloaded': False
+
+    }
+
+    if no_sections != 0:
+        exists_already = check_existence(sections, db)
+    else:
+        sections_available = False
+
+    if not exists_already and sections_available:
+        task = celery.send_task(task_name, kwargs=kwargs, serializer='json')
 
+    if sections_available and not exists_already:
+        status = "success"
 
-    task = celery.send_task(task_name, kwargs = kwargs, serializer='json')
+    elif not sections_available:
+        status = "no_sections"
+
+    elif exists_already:
+        status = "pptx_exists"
 
     package = {
-        'taskID': task.id,
-        'sections': sections
+        'taskID': custom_id,
+        'sections': sections,
+        'status': status
     }
 
-
     return JSONResponse(package)
-
-
-
-
+
+
+def check_existence(sections, db):
+    exists_already = False
+    no_sections = len(sections)
+    query = {"kwargs.sections": {"$size": no_sections, "$all": sections}}
+
+    hits = db.count_documents(query)
+
+    if hits > 0:
+        exists_already = True
+
+    return exists_already
+
+
+@app.post("/v1/download", tags=["powerpoint"])
+async def download_pptx(download: Download):
+
+    task_id = download.taskID
+
+    result = db.find_one({"kwargs.customID": task_id}, {'result': 1, '_id': 0})
+    unpack = result["result"]
+    unpack = json.loads(unpack)
+    file_path = unpack["filePath"]
+
+    # return file_path
+    return FileResponse(file_path)

backend/req.txt

@@ -1,27 +1,26 @@
-certifi==2020.6.20
-click==7.1.2
-fastapi==0.61.2
-h11==0.11.0
-lxml==4.6.1
-Pillow==8.0.1
-pydantic==1.7.2
-python-pptx==0.6.18
-starlette==0.13.6
-typing==3.7.4.3
-uvicorn==0.12.2
-XlsxWriter==1.3.7
 aiofiles==0.6.0
 amqp==5.0.2
 billiard==3.6.3.0
 celery==5.0.3
+certifi==2020.6.20
 click==7.1.2
 click-didyoumean==0.0.3
 click-plugins==1.1.1
 click-repl==0.1.6
+fastapi==0.61.2
+h11==0.11.0
 kombu==5.0.2
+lxml==4.6.1
+Pillow==8.0.1
 prompt-toolkit==3.0.8
+pydantic==1.7.2
+pymongo==3.11.2
+python-pptx==0.6.18
 pytz==2020.4
-redis==3.5.3
 six==1.15.0
+starlette==0.13.6
+typing==3.7.4.3
+uvicorn==0.12.2
 vine==5.0.0
-wcwidth==0.2.5
+wcwidth==0.2.5
+XlsxWriter==1.3.7

backend/server.py

@@ -2,10 +2,14 @@
 from celery import Celery
 
 CELERY_BROKER_URL = os.getenv("RABBITMQ")
-CELERY_RESULT_BACKEND = os.getenv("REDISSERVER")
+CELERY_RESULT_BACKEND = os.getenv("MONGODB")
 
 celery = Celery("worker", backend=CELERY_RESULT_BACKEND, broker=CELERY_BROKER_URL)
 
 celery.conf.update(
-   result_extended=True
+   result_extended=True,
+   mongodb_backend_settings={
+        'database': 'taskdb',
+        'taskmeta_collection': 'ta',
+    }
 )

celery/.gitignore

@@ -0,0 +1 @@
+env/