init

.gitignore

@@ -0,0 +1,2 @@
+/logs
+__pycache__

config.json

@@ -0,0 +1,22 @@
+{
+    "abp": {
+      "api_base": "https://stag-abp-api.iconsz.com",
+      "tenant": "ics",
+      "username": "ics",
+      "password": "1qaz@WSX",
+      "client_id": "engine_CrawlerEngines",
+      "client_secret": "7ii7U9AabN2*CrawlerEngines",    
+      "scope": "FX"
+    },
+    "engine": {
+      
+    },
+    "queue_processor": {
+      "get_pending_queue_interval_seconds": 2
+    },
+    "queue_client": {
+      "check_queue_interval_seconds": 1,
+      "call_wait_return_default_timeout_seconds": 60
+    }
+  }
+  

engines/crawler_google_search.py

@@ -0,0 +1,6 @@
+class CrawlerGoogleSearch:
+    def process(inputData):
+        print("CrawlerGoogleSearch / inputData", inputData)
+        # TODO 具体的实现逻辑
+
+        return {"full_name": "xxx", "date_of_birth": "1956-01-01"}

iCloudEngine

@@ -0,0 +1 @@
+Subproject commit 646c4424169c93da36fdf568904efeaf644af9b4

iCloudEngine.git.bat

@@ -0,0 +1,2 @@
+git clone https://gitea.iconsz.com/iCON/iCloudEngine.git
+pause

main.py

@@ -0,0 +1,46 @@
+from engines.crawler_google_search import CrawlerGoogleSearch
+from utils.logger import logger
+import os, sys, time, traceback, json
+
+base_dir = os.path.dirname(os.path.abspath(__file__))
+relative_path = os.path.join(base_dir, "iCloudEngine/src")
+sys.path.append(relative_path)
+
+from queue_processor import QueueProcessor
+
+
+class Main(QueueProcessor):
+    def processor_handle(self, input):
+        print("input:", input)  # {'keyword': '林郑月娥' }
+
+        currentEngineId = input["currentEngineId"]
+        inputData = json.loads(input["inputData"])
+        # keyword = inputData["keyword"]
+        # print("keyword:", keyword)
+        match currentEngineId:
+            case 9000:  # crawler_bbc_search
+                return CrawlerGoogleSearch.process(inputData)
+            case 10000:  # crawler_bbc_search
+                print(2)
+            case 11000:  # crawler_wikipedia
+                print(3)
+            case 12000:  # crawler_webb_site
+                print(4)
+
+
+if __name__ == "__main__":
+    try:
+        # Main().startV2([9000])
+        Main().startV2()
+
+    except Exception as e:
+        logger.warning(f"excute exception:{e}")
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        traceback_details = traceback.extract_tb(exc_traceback)
+        filename, line_number, function_name, text = traceback_details[-1]
+        logger.error(f"Exception occurred in {filename} at line {line_number}: {e}")
+        logger.error(f"Function name: {function_name}")
+        logger.error(f"Text: {text}")
+
+    finally:
+        logger.warning("application completed")

submit_test.py

@@ -0,0 +1,27 @@
+import os, sys
+
+base_dir = os.path.dirname(os.path.abspath(__file__))
+relative_path = os.path.join(base_dir, "iCloudEngine/src")
+sys.path.append(relative_path)
+
+from queue_client import QueueClient
+
+
+class submit_test:
+    def submit(code, keyword):
+        client = QueueClient()
+        returnData = client.call(code, {"keyword": keyword})
+        print(returnData)
+
+
+# 创建一个测试队列：9000 / GoogleSearch
+submit_test.submit(9000, "林郑月娥")
+
+# 创建一个测试队列：10000 / BBCSearch
+submit_test.submit(10000, "林郑月娥")
+
+# 创建一个测试队列：11000 / Wikipedia
+submit_test.submit(11000, "林郑月娥")
+
+# 创建一个测试队列：12000 / WebbSite
+submit_test.submit(12000, "林郑月娥")

utils/logger.py

@@ -0,0 +1,42 @@
+import logging
+import os
+from datetime import datetime
+import requests
+
+if not os.path.exists("logs"):
+    os.makedirs("logs")
+
+# 配置日志的基本设置
+logging.basicConfig(
+    filename=os.path.join("logs", datetime.now().strftime("%Y-%m-%d") + ".log"),
+    level=logging.WARNING,
+    format="%(asctime)s:%(levelname)s:%(message)s",
+    filemode="a",
+    encoding="utf-8",
+)
+
+# 设置requests库的日志级别为WARNING或更高，以确保不记录INFO或DEBUG级别的日志
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("elasticsearch").setLevel(logging.WARNING)
+
+
+class logger:
+    # def debug(msg: str):
+    #     logging.debug(msg)
+
+    # def info(msg: str):
+    # logging.info(msg)
+
+    def warning(msg, printLog=True):
+        logging.warning(msg)
+        if printLog == True:
+            print(msg)
+
+    def error(msg: str, printLog=True):
+        logging.error(msg)
+        if printLog == True:
+            print(msg)
+
+    # def critical(msg: str):
+    #     logging.critical(msg)