Skip to content

Commit e9067c8

Browse files
JonoYangJonoYang
committed
Add scripts from scancode-toolkit/etc/release/ #33
Signed-off-by: Jono Yang <[email protected]>
1 parent 1bcaaa5 commit e9067c8

19 files changed

+4518
-0
lines changed

Diff for: ‎etc/scripts/bootstrap.py

+212
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
#
4+
# Copyright (c) nexB Inc. and others. All rights reserved.
5+
# ScanCode is a trademark of nexB Inc.
6+
# SPDX-License-Identifier: Apache-2.0
7+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
8+
# See https://github.com/nexB/scancode-toolkit for support or download.
9+
# See https://aboutcode.org for more information about nexB OSS projects.
10+
#
11+
12+
import itertools
13+
14+
import click
15+
16+
import utils_thirdparty
17+
from utils_thirdparty import Environment
18+
from utils_thirdparty import PypiPackage
19+
20+
21+
@click.command()
22+
23+
@click.option('-r', '--requirements-file',
24+
type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
25+
metavar='FILE',
26+
multiple=True,
27+
default=['requirements.txt'],
28+
show_default=True,
29+
help='Path to the requirements file(s) to use for thirdparty packages.',
30+
)
31+
@click.option('-d', '--thirdparty-dir',
32+
type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
33+
metavar='DIR',
34+
default=utils_thirdparty.THIRDPARTY_DIR,
35+
show_default=True,
36+
help='Path to the thirdparty directory where wheels are built and '
37+
'sources, ABOUT and LICENSE files fetched.',
38+
)
39+
@click.option('-p', '--python-version',
40+
type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
41+
metavar='PYVER',
42+
default=utils_thirdparty.PYTHON_VERSIONS,
43+
show_default=True,
44+
multiple=True,
45+
help='Python version(s) to use for this build.',
46+
)
47+
@click.option('-o', '--operating-system',
48+
type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
49+
metavar='OS',
50+
default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
51+
multiple=True,
52+
show_default=True,
53+
help='OS(ses) to use for this build: one of linux, mac or windows.',
54+
)
55+
@click.option('-l', '--latest-version',
56+
is_flag=True,
57+
help='Get the latest version of all packages, ignoring version specifiers.',
58+
)
59+
@click.option('--sync-dejacode',
60+
is_flag=True,
61+
help='Synchronize packages with DejaCode.',
62+
)
63+
@click.option('--with-deps',
64+
is_flag=True,
65+
help='Also include all dependent wheels.',
66+
)
67+
@click.help_option('-h', '--help')
68+
def bootstrap(
69+
requirements_file,
70+
thirdparty_dir,
71+
python_version,
72+
operating_system,
73+
with_deps,
74+
latest_version,
75+
sync_dejacode,
76+
build_remotely=False,
77+
):
78+
"""
79+
Boostrap a thirdparty Python packages directory from pip requirements.
80+
81+
Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for
82+
the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible
83+
with all the provided ``--python-version`` PYVER(s) and ```--operating_system``
84+
OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and
85+
.LICENSE files.
86+
87+
Optionally ignore version specifiers and use the ``--latest-version``
88+
of everything.
89+
90+
Sources and wheels are fetched with attempts first from PyPI, then our remote repository.
91+
If missing wheels are built as needed.
92+
"""
93+
# rename variables for clarity since these are lists
94+
requirements_files = requirements_file
95+
python_versions = python_version
96+
operating_systems = operating_system
97+
98+
# create the environments we need
99+
evts = itertools.product(python_versions, operating_systems)
100+
environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
101+
102+
# collect all packages to process from requirements files
103+
# this will fail with an exception if there are packages we cannot find
104+
105+
required_name_versions = set()
106+
107+
for req_file in requirements_files:
108+
nvs = utils_thirdparty.load_requirements(
109+
requirements_file=req_file, force_pinned=False)
110+
required_name_versions.update(nvs)
111+
if latest_version:
112+
required_name_versions = set((name, None) for name, _ver in required_name_versions)
113+
114+
print(f'PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES')
115+
116+
# fetch all available wheels, keep track of missing
117+
# start with local, then remote, then PyPI
118+
119+
print('==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS')
120+
# list of all the wheel filenames either pre-existing, fetched or built
121+
# updated as we progress
122+
available_wheel_filenames = []
123+
124+
local_packages_by_namever = {
125+
(p.name, p.version): p
126+
for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir)
127+
}
128+
129+
# list of (name, version, environment) not local and to fetch
130+
name_version_envt_to_fetch = []
131+
132+
# start with a local check
133+
for (name, version), envt in itertools.product(required_name_versions, environments):
134+
local_pack = local_packages_by_namever.get((name, version,))
135+
if local_pack:
136+
supported_wheels = list(local_pack.get_supported_wheels(environment=envt))
137+
if supported_wheels:
138+
available_wheel_filenames.extend(w.filename for w in supported_wheels)
139+
print(f'====> No fetch or build needed. '
140+
f'Local wheel already available for {name}=={version} '
141+
f'on os: {envt.operating_system} for Python: {envt.python_version}')
142+
continue
143+
144+
name_version_envt_to_fetch.append((name, version, envt,))
145+
146+
print(f'==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS')
147+
148+
# list of (name, version, environment) not fetch and to build
149+
name_version_envt_to_build = []
150+
151+
# then check if the wheel can be fetched without building from remote and Pypi
152+
for name, version, envt in name_version_envt_to_fetch:
153+
154+
fetched_fwn = utils_thirdparty.fetch_package_wheel(
155+
name=name,
156+
version=version,
157+
environment=envt,
158+
dest_dir=thirdparty_dir,
159+
)
160+
161+
if fetched_fwn:
162+
available_wheel_filenames.append(fetched_fwn)
163+
else:
164+
name_version_envt_to_build.append((name, version, envt,))
165+
166+
# At this stage we have all the wheels we could obtain without building
167+
for name, version, envt in name_version_envt_to_build:
168+
print(f'====> Need to build wheels for {name}=={version} on os: '
169+
f'{envt.operating_system} for Python: {envt.python_version}')
170+
171+
packages_and_envts_to_build = [
172+
(PypiPackage(name, version), envt)
173+
for name, version, envt in name_version_envt_to_build
174+
]
175+
176+
print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS')
177+
178+
package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels(
179+
packages_and_envts=packages_and_envts_to_build,
180+
build_remotely=build_remotely,
181+
with_deps=with_deps,
182+
dest_dir=thirdparty_dir,
183+
)
184+
if wheel_filenames_built:
185+
available_wheel_filenames.extend(available_wheel_filenames)
186+
187+
for pack, envt in package_envts_not_built:
188+
print(
189+
f'====> FAILED to build any wheel for {pack.name}=={pack.version} '
190+
f'on os: {envt.operating_system} for Python: {envt.python_version}'
191+
)
192+
193+
print(f'==> FETCHING SOURCE DISTRIBUTIONS')
194+
# fetch all sources, keep track of missing
195+
# This is a list of (name, version)
196+
utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
197+
198+
print(f'==> FETCHING ABOUT AND LICENSE FILES')
199+
utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
200+
201+
############################################################################
202+
if sync_dejacode:
203+
print(f'==> SYNC WITH DEJACODE')
204+
# try to fetch from DejaCode any missing ABOUT
205+
# create all missing DejaCode packages
206+
pass
207+
208+
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
209+
210+
211+
if __name__ == '__main__':
212+
bootstrap()

Diff for: ‎etc/scripts/build_wheels.py

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
#
4+
# Copyright (c) nexB Inc. and others. All rights reserved.
5+
# ScanCode is a trademark of nexB Inc.
6+
# SPDX-License-Identifier: Apache-2.0
7+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
8+
# See https://github.com/nexB/scancode-toolkit for support or download.
9+
# See https://aboutcode.org for more information about nexB OSS projects.
10+
#
11+
import click
12+
13+
import utils_thirdparty
14+
15+
16+
@click.command()
17+
18+
@click.option('-n', '--name',
19+
type=str,
20+
metavar='PACKAGE_NAME',
21+
required=True,
22+
help='Python package name to add or build.',
23+
)
24+
@click.option('-v', '--version',
25+
type=str,
26+
default=None,
27+
metavar='VERSION',
28+
help='Python package version to add or build.',
29+
)
30+
@click.option('-d', '--thirdparty-dir',
31+
type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
32+
metavar='DIR',
33+
default=utils_thirdparty.THIRDPARTY_DIR,
34+
show_default=True,
35+
help='Path to the thirdparty directory where wheels are built.',
36+
)
37+
@click.option('-p', '--python-version',
38+
type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
39+
metavar='PYVER',
40+
default=utils_thirdparty.PYTHON_VERSIONS,
41+
show_default=True,
42+
multiple=True,
43+
help='Python version to use for this build.',
44+
)
45+
@click.option('-o', '--operating-system',
46+
type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
47+
metavar='OS',
48+
default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
49+
multiple=True,
50+
show_default=True,
51+
help='OS to use for this build: one of linux, mac or windows.',
52+
)
53+
@click.option('--build-remotely',
54+
is_flag=True,
55+
help='Build missing wheels remotely.',
56+
)
57+
@click.option('--with-deps',
58+
is_flag=True,
59+
help='Also include all dependent wheels.',
60+
)
61+
@click.option('--verbose',
62+
is_flag=True,
63+
help='Provide verbose output.',
64+
)
65+
@click.help_option('-h', '--help')
66+
def build_wheels(
67+
name,
68+
version,
69+
thirdparty_dir,
70+
python_version,
71+
operating_system,
72+
with_deps,
73+
build_remotely,
74+
verbose,
75+
):
76+
"""
77+
Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and
78+
optional VERSION. Build wheels compatible with all the `--python-version`
79+
PYVER(s) and `--operating_system` OS(s).
80+
81+
Build native wheels remotely if needed when `--build-remotely` and include
82+
all dependencies with `--with-deps`.
83+
"""
84+
utils_thirdparty.add_or_upgrade_built_wheels(
85+
name=name,
86+
version=version,
87+
python_versions=python_version,
88+
operating_systems=operating_system,
89+
dest_dir=thirdparty_dir,
90+
build_remotely=build_remotely,
91+
with_deps=with_deps,
92+
verbose=verbose,
93+
)
94+
95+
96+
if __name__ == '__main__':
97+
build_wheels()

Diff for: ‎etc/scripts/check_thirdparty.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
#
4+
# Copyright (c) nexB Inc. and others. All rights reserved.
5+
# ScanCode is a trademark of nexB Inc.
6+
# SPDX-License-Identifier: Apache-2.0
7+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
8+
# See https://github.com/nexB/scancode-toolkit for support or download.
9+
# See https://aboutcode.org for more information about nexB OSS projects.
10+
#
11+
import click
12+
13+
import utils_thirdparty
14+
15+
16+
@click.command()
17+
18+
@click.option('-d', '--thirdparty-dir',
19+
type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
20+
required=True,
21+
help='Path to the thirdparty directory to check.',
22+
)
23+
@click.help_option('-h', '--help')
24+
def check_thirdparty_dir(thirdparty_dir):
25+
"""
26+
Check a thirdparty directory for problems.
27+
"""
28+
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
29+
30+
31+
if __name__ == '__main__':
32+
check_thirdparty_dir()

0 commit comments

Comments
 (0)