From dda95f87ceb384a8c3e10df8fd0429f26590f898 Mon Sep 17 00:00:00 2001
From: wangxiyuan <wangxiyuan1007@gmail.com>
Date: Thu, 6 May 2021 16:07:30 +0800
Subject: [PATCH] Add dependence fetch tool

---
 README.en.md    |   1 +
 README.md       |   1 +
 tools/fetch_dep | 190 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 192 insertions(+)
 create mode 100755 tools/fetch_dep

diff --git a/README.en.md b/README.en.md
index e9251d1..03e4d7e 100644
--- a/README.en.md
+++ b/README.en.md
@@ -53,6 +53,7 @@ Meeting agenda and summary: <https://etherpad.openeuler.org/p/sig-openstack-meet
 |   └── architecture.md                           "RPM subcontracting rules"
 └── tools                                         "openstack packaging, dependency analysis, etc"
 |   └── pyporter                                  "Python library RPM spec and package generation work"
+|   └── fetch_dep                                 "The dependence analysis tool for OpenStack service"
 └── README.md                                     "SIG Document Entry(Chinese)"
 └── README.en.md                                  "SIG Document Entry(English)"
 ```
diff --git a/README.md b/README.md
index 9239135..46d93ed 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ openEuler OpenStack SIG致力于结合多样性算力为openstack社区贡献更
 |   └── architecture.md                           "RPM分包规则"
 └── tools                                         "openstack打包、依赖分析等工作"
 |   └── pyporter                                  "python库的RPM spec和软件包生成工作"
+|   └── fetch_dep                                 "OpenStack服务依赖库分析工具"
 └── README.md                                     "SIG文档入口(中文版)"
 └── README.en.md                                  "SIG文档入口(英文版)"
 ```
diff --git a/tools/fetch_dep b/tools/fetch_dep
new file mode 100755
index 0000000..6495122
--- /dev/null
+++ b/tools/fetch_dep
@@ -0,0 +1,190 @@
+#!/usr/bin/python3
+#
+# The tool for fetching the minimal required version of the dependencies for openstack service.
+
+# Update: 2021-05-06
+#
+# This tool need python3 support. And before use this tool, please install the required packages first:
+#     pip(or pip3) install packaging requests
+#
+# How to use
+#    1. ./fetch_dep rocky
+#        fetch the dependencies of openstack rocky release from pypi, save the result to "result.csv"
+#    2. ./fetch_dep -s rocky
+#        fetch the dependencies of openstack rocky release from local cache, save the result to "result.csv"
+#    3. ./fetch_dep -s -o openstack_v victoria
+#        fetch the dependencies of openstack rocky release from local cache, save the result to "openstack_v.csv"
+
+import argparse
+import csv
+import json
+import os
+from packaging import version as p_version
+from pathlib import Path
+import re
+import requests
+
+
+PROJECT_JSON_URL = "https://pypi.org/pypi/%s/%s/json"
+
+# TODO: Add more service and release
+OPENSTACK_RELEASE_MAP = {
+    "rocky": {
+        "cinder": "13.0.9",
+        "glance": "17.0.1",
+        "horizon": "14.1.0",
+        "ironic": "11.1.4",
+        "keystone": "14.2.0",
+        "neutron": "13.0.7",
+        "nova": "18.3.0",
+        "python-openstackclient": "3.16.3",
+        "tempest": "19.0.0",
+    },
+    "victoria": {
+        "cinder": "17.1.0",
+        "glance": "21.0.0",
+        "horizon": "18.6.2",
+        "ironic": "16.0.3",
+        "keystone": "18.0.0",
+        "neutron": "17.1.1",
+        "nova": "22.2.0",
+        "python-openstackclient": "5.4.0",
+        "tempest": "25.0.1",
+    }
+}
+
+
+# TODO: Add maximun version of the dependencies
+class FetchDependence(object):
+    def __init__(self, openstack_release, output):
+        self.openstack_release = openstack_release
+        self.project_dict = OPENSTACK_RELEASE_MAP[openstack_release]
+        self.dep_list = {}
+        self.output = output + ".csv" if not output.endswith(".csv") else output
+        self.cache_path = "./pypi_json_%s/" % self.openstack_release
+        path = Path(self.cache_path)
+        if not path.exists():
+            path.mkdir()
+
+    def _fetch_json(self, project, version, is_init):
+        """Fetch package json file from pypi or local cache"""
+        if is_init:
+            file_path = (self.cache_path + "%s-%s.json") % (project, version)
+            if Path(file_path).exists():
+                with open(file_path, 'r', encoding='utf8') as fp:
+                    json_data = json.load(fp)
+            else:
+                for i in range(5):
+                    try:
+                        content = requests.get(PROJECT_JSON_URL % (project, version)).content.decode()
+                        json_data = json.loads(content)
+                        break
+                    except Exception:
+                        continue
+                else:
+                    raise Exception("Network is unstable, please try again")
+                with open(file_path, 'a', encoding='utf8') as fp:
+                    json.dump(json_data, fp, ensure_ascii=False)
+        else:
+            file_path = "./pypi_json_%s/" % self.openstack_release
+            file_list = os.listdir(file_path)
+            for project_file in file_list:
+                if re.match("^%s-[0-9]+" % project, project_file):
+                    with open(file_path+project_file, 'r', encoding='utf8') as fp:
+                        json_data = json.load(fp)
+                        break
+            else:
+                raise Exception("Please init first.")
+        return json_data
+
+    def _get_dep(self, project, version, is_init):
+        """Count dependencies by recursion way"""
+        project_json = self._fetch_json(project, version, is_init)
+        project_dep_list = project_json['info']['requires_dist']
+        if not project_dep_list:
+            return
+        for item in project_dep_list:
+            idx = item.find(';')
+            if idx != -1:
+                item = item[:idx]
+            try:
+                project, version = item.split(' ', 1)
+                project = project.split("[")[0]
+                if '==' in version:
+                    version = version.split('==')[-1].strip(' ')[:-1].split(',')[0]
+                elif '>=' in version:
+                    version = version.split('>=')[-1].strip(' ')[:-1].split(',')[0]
+                elif '<' in version:
+                    version = version.split('<')[-1].strip(' ')[:-1].split(',')[0]
+            except ValueError:
+                project, version = item, "N/A"
+            if version not in ['N/A', "0a"]:
+                # TODO: Add special project list
+                if not self.dep_list.get(project) or p_version.parse(version) > p_version.parse(self.dep_list[project]):
+                    if project.lower() == 'prettytable' and p_version.parse(version) < p_version.parse('0.7.1'):
+                        version = '0.7.1'
+                    if project.lower() == 'msgpack' and p_version.parse(version) < p_version.parse('0.5.0'):
+                        version = '0.5.0'
+                    self.dep_list[project] = version
+                    self._get_dep(project, version, is_init)
+
+    def _purge(self):
+        """Purge the duplicated json file"""
+        file_path = "./pypi_json_%s/" % self.openstack_release
+        file_list = os.listdir(file_path)
+        file_dict = {}
+        for file_name in file_list:
+            if not file_name.endswith("json"):
+                continue
+            project_name, project_version = file_name.rsplit('-', 1)[0], file_name.rsplit('-', 1)[1].rsplit('.', 1)[0]
+            if not file_dict.get(project_name):
+                file_dict[project_name] = project_version
+            elif p_version.parse(project_version) < p_version.parse(file_dict[project_name]):
+                os.remove(file_path+file_name)
+            elif p_version.parse(project_version) > p_version.parse(file_dict[project_name]):
+                os.remove(file_path+project_name+"-"+file_dict[project_name]+".json")
+                file_dict[project_name] = project_version
+
+    def init_all_dep(self):
+        """download all related json file of packages"""
+        self.dep_list = {}
+        for project, version in self.project_dict.items():
+            self._get_dep(project, version, True)
+        self._purge()
+
+    def get_all_dep(self):
+        """fetch all related dependent packages"""
+        self.dep_list = {}
+        for project, version in self.project_dict.items():
+            self._get_dep(project, version, False)
+
+        with open(self.output, "w") as csv_file:
+            writer=csv.writer(csv_file)
+            writer.writerow(["Project", "Version"])
+            for key, value in self.dep_list.items():
+                writer.writerow([key, value])
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-o", "--output", help="Output file name, default: result", type=str, default="result")
+    parser.add_argument("-s", "--skip-init", help="Skip init step", action="store_true")
+    parser.add_argument("release", type=str, help="OpenStack release name", choices=OPENSTACK_RELEASE_MAP.keys())
+
+    args = parser.parse_args()
+
+    myobj = FetchDependence(args.release, args.output)
+
+    if not args.skip_init:
+        print("Start fetch caching files to pypi_json_%s folder" % args.release)
+        print("...")
+        myobj.init_all_dep()
+        print("Success cached pypi files.")
+        print("...")
+        print("Note: There is no need to fetch pypi again from next time, unless you want to refresh the cache. "
+              "Please use -s to skip this step.")
+        print("...")
+    print("Start count dependencies")
+    print("...")
+    myobj.get_all_dep()
+    print("Success fetched dependencies, the result is saved into %s file" % args.output)
-- 
Gitee