init

Author: microhhh

.gitignore

@@ -0,0 +1,7 @@
+*.pyc
+*__pycache__
+build/
+dist/
+dwf_sdk_python.egg-info/
+.idea/
+*run.log

README.md

@@ -1,2 +1 @@
 # dwf-sdk-python
-dwf-sdk-python

dwf.conf

@@ -0,0 +1,27 @@
+[HDFS]
+DWF_MODEL_STORAGE_NAME = MODEL_STORAGE
+DWF_MODEL_LOG_STORAGE_NAME = LOG_STORAGE
+
+[LOCAL]
+LOCAL_TMP_DIR = /tmp
+
+[LOG]
+LOG_FILE = run.log
+LOG_LEVEL = debug
+
+[DB]
+DB_HOST = 192.168.6.106
+DB_PORT = 5432
+DB_NAME = dwf_test
+DB_USER = postgres
+DB_PASSWORD = xlearn123
+
+[TEST_DB]
+DB_HOST = 192.168.6.106
+DB_PORT = 5432
+DB_NAME = dwf_test
+DB_USER = postgres
+DB_PASSWORD = xlearn123
+
+[ROLE]
+ADMIN_ID = USERaaaaaaaaaaaaaaaaaaaaaaaaaaaa

dwf/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['algorithm', 'common', 'datapattern', 'dataset', 'datasource', 'model', 'package', 'util']

dwf/algorithm/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['metadata']

dwf/algorithm/metadata/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding:utf-8 -*-
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['algorithm_crud']

dwf/algorithm/metadata/algorithm_crud.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['config', 'exception', 'log']

dwf/common/__init__.py

@@ -0,0 +1,30 @@
+# -*- coding:utf-8 -*-
+#
+# DataWay Dataset SDK
+# Initial Date: 2018.06.15
+#
+# Title: Global Configuration
+#
+# Version 0.1
+#
+
+import configparser
+import os
+
+# don't add logger here to avoid cycled dependencies.
+
+deploy_config = configparser.ConfigParser()
+file_name = "dwf.conf"
+if not os.path.exists(file_name):
+    file_name = "../dwf.conf"
+if not os.path.exists(file_name):
+    file_name = "/etc/dwf/dwf.conf"
+deploy_config.read(file_name)
+
+test_config = configparser.ConfigParser()
+file_name = "dwf.conf"
+if not os.path.exists(file_name):
+    file_name = "../dwf.conf"
+if not os.path.exists(file_name):
+    file_name = "/etc/dwf/dwf.conf"
+test_config.read(file_name)

dwf/common/config.py

@@ -0,0 +1,47 @@
+#
+# DataWay SDK
+# Author: peizhongyi([email protected])
+# Initial Date: 2018.06.16
+#
+# Title: DWFException
+#
+# Version 0.1
+#
+
+
+class DWFException(Exception):
+    def __init__(self, status, msg):
+        self.STATUS = status
+        self.MSG = msg
+        Exception.__init__(self, msg)
+
+
+'''
+Unknown Exceptions
+'''
+UNDEFINE = DWFException(1, '发现未定义异常')
+
+'''
+Exceptions for Parameters
+'''
+PARAM_LACK = DWFException(2001, '缺少参数')
+PARAM_WRONG_TYPE = DWFException(2002, '参数类型错误')
+
+'''
+Exceptions for Database
+'''
+DATABASE_CONN_FAIL = DWFException(3001, '数据库连接错误')
+NUM_RESULT_MISMATCH = DWFException(3002, '查询记录结果数不符合预期')
+ILLEGAL_REPEATED_FILED = DWFException(3003, '字段不可重复')
+NON_EXISTING_ALGORITHM = DWFException(3004, '算法不存在')
+NON_EXISTING_DATA_PATTERN = DWFException(3004, '数据模式不存在')
+'''
+Exceptions for Files
+'''
+FILE_NOT_EXIST = DWFException(4001, '文件不存在')
+FILE_FORMAT_ERROR = DWFException(4002, '文件格式不符合预期')
+
+'''
+Exceptions for Development
+'''
+NOT_IMPLEMENTED = DWFException(5001, '方法未实现')

dwf/common/exception.py

@@ -0,0 +1,21 @@
+# -*- coding:utf-8 -*-
+#
+# DataWay Dataset SDK
+# Author: peizhongyi([email protected])
+# Initial Date: 2018.06.16
+#
+# Title: Logger
+#
+# Version 0.1
+#
+
+import logging as logger
+from dwf.common.config import deploy_config
+
+LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
+LOG_FILE = deploy_config.get('LOG', 'LOG_FILE')
+
+if deploy_config.get('LOG', 'LOG_LEVEL') == 'debug':
+    logger.basicConfig(filename=LOG_FILE, level=logger.DEBUG, format=LOG_FORMAT)
+else:
+    logger.basicConfig(filename=LOG_FILE, level=logger.INFO, format=LOG_FORMAT)

dwf/common/log.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['metadata']

dwf/datapattern/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)
+
+__all__ = ['datapattern_crud']

dwf/datapattern/metadata/__init__.py

@@ -0,0 +1,97 @@
+# -*- coding:utf-8 -*-
+#
+# DataWay Algorithm SDK
+# Initial Date: 2018.10.27
+#
+# Title: methods for metadata of pattern
+#
+# Version 1.0
+#
+
+from dwf.ormmodels import *
+from dwf.common.log import logger
+from dwf.common.exception import *
+from dwf.util.id import generate_primary_key
+
+
+class DataPatternCRUD:
+
+    def __init__(self, db_session):
+        self.db_session = db_session
+
+    def add_datapattern(self, name, semantic=None):
+        #    NO.A317
+        #        Wait for jinying
+        #    Args：
+        #        package_name - The name of package.
+        #        deployment_path - The path to deploy algorithm.
+        #        upload_source - The upload source of package.
+        #        requirement_list - Dependencies of algorithms in this package.
+        #    Returns：
+        #        package_id
+        #        hyperparameter_config
+        #    Exceptions:
+        #        ADD_FAILURE
+        id = generate_primary_key('DPAT')
+        pattern = DataPattern(id=id, name=name, semantic=semantic)
+        self.db_session.add(pattern)
+        self.db_session.commit()
+        return id
+
+    def delete_datapattern(self, delete_pattern_id):
+        #    NO.A320
+        #        Wait for jinying
+        #    Args：
+        #        package_id: The id of package to be deleted.
+        #    Returns：
+        #        None.
+        #    Exceptions:
+        #        NON_EXISTING_PACKAGE - The given package_id does not exist.
+        #        DELETE_FAILURE - The package cannot be deleted.
+        pending = self.db_session.query(DataPattern).get(delete_pattern_id)
+        self.db_session.delete(pending)
+        self.db_session.commit()
+        return
+
+    def update_datapattern(self, delete_pattern_id, name=None, semantic=None):
+        #    NO.A321
+        #        Wait for jinying
+        #    Args：
+        #        package_name - The name of package.
+        #        deployment_path - The path to deploy algorithm.
+        #        upload_source - The upload source of package.
+        #        requirement_list - Dependencies of algorithms in this package.
+        #    Returns：
+        #        package_id - The id of package.
+        #        conda_id - The id of conda.
+        #    Exceptions:
+        #        UPDATE_FAILURE - Fail to add this package.
+        pending = self.db_session.query(DataPattern).get(delete_pattern_id)
+        if delete_pattern_id is None:
+            logger.error('delete_pattern_id is needed')
+            raise PARAM_LACK
+
+        if name is not None:
+            pending.plt_name = name
+        if semantic is not None:
+            pending.semantic = semantic
+        self.db_session.commit()
+        return pending
+
+    def query_datapattern(self, delete_pattern_id):
+        #    NO.A322
+        #        Wait for jinying
+        #    Args：
+        #        package_id: The id of package.
+        #    Returns：
+        #        {
+        #            package_name:"***",
+        #            deployment_path:"***",
+        #            upload_source:"***",
+        #            requirement_list:"***"
+        #        }
+        pending = self.db_session.query(DataPattern).get(delete_pattern_id)
+        if pending is None:
+            logger.error('data pattern is not found')
+            raise NON_EXISTING_DATA_PATTERN
+        return pending

dwf/datapattern/metadata/datapattern_crud.py

@@ -0,0 +1,7 @@
+# -*- coding:utf-8 -*-
+
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+
+__all__ = ['io','metadata']