Skip to content

Commit b77a55b

Browse files
jonathanmetzmanjonathanmetzman
authored
[ClusterFuzzLite] Support GCB and gsutil/gcs as filestore. (#6629)
* add gsutil filestore * lint * Fix * Add build image script * get gcb fuzzing working * fmt and fix config_utils_test * Check that crashes are uploaded * Add no_filestore * fix test * fix tests * fix * Print crash URL * Fix * fix * fmt * lnt * fix * fmt
1 parent d951635 commit b77a55b

20 files changed

+313
-60
lines changed

‎.dockerignore

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.git
2+
.venv
23
infra/cifuzz/test_data/*
34
docs/*
45

@@ -8,4 +9,6 @@ docs/*
89
build
910
*~
1011
.DS_Store
11-
*.swp
12+
*.swp
13+
.pytest_cache
14+
*__pycache__/*

‎infra/cifuzz/build-images.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#! /bin/bash -eux
2+
# Copyright 2021 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Script for building the docker images for cifuzz.
17+
18+
CIFUZZ_DIR=$(dirname "$0")
19+
CIFUZZ_DIR=$(realpath $CIFUZZ_DIR)
20+
INFRA_DIR=$(realpath $CIFUZZ_DIR/..)
21+
OSS_FUZZ_ROOT=$(realpath $INFRA_DIR/..)
22+
23+
# Build cifuzz-base.
24+
docker build --tag gcr.io/oss-fuzz-base/cifuzz-base --file $CIFUZZ_DIR/cifuzz-base/Dockerfile $OSS_FUZZ_ROOT
25+
26+
# Build run-fuzzers and build-fuzzers images.
27+
docker build --tag gcr.io/oss-fuzz-base/cifuzz-build-fuzzers:v1 --file $INFRA_DIR/build_fuzzers.Dockerfile $INFRA_DIR
28+
docker build --tag gcr.io/oss-fuzz-base/cifuzz-run-fuzzers:v1 --file $INFRA_DIR/run_fuzzers.Dockerfile $INFRA_DIR

‎infra/cifuzz/build_fuzzers_test.py

+1
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ def test_external_generic_project(self):
201201
project_repo_name=project_repo_name,
202202
workspace=self.workspace,
203203
git_url=git_url,
204+
filestore='no_filestore',
204205
commit_sha='HEAD',
205206
project_src_path=project_src_path,
206207
base_commit='HEAD^1')

‎infra/cifuzz/cifuzz-base/Dockerfile

+10-1
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,17 @@ RUN apt-get update && \
2020
apt-get install -y systemd && \
2121
apt-get install -y --no-install-recommends nodejs npm && \
2222
wget https://download.docker.com/linux/ubuntu/dists/focal/pool/stable/amd64/docker-ce-cli_20.10.8~3-0~ubuntu-focal_amd64.deb -O /tmp/docker-ce.deb && \
23-
dpkg -i /tmp/docker-ce.deb && rm /tmp/docker-ce.deb
23+
dpkg -i /tmp/docker-ce.deb && \
24+
rm /tmp/docker-ce.deb && \
25+
mkdir -p /opt/gcloud && \
26+
wget -qO- https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar zxv -C /opt/gcloud && \
27+
/opt/gcloud/google-cloud-sdk/install.sh --usage-reporting=false --bash-completion=false --disable-installation-options && \
28+
apt-get -y install gcc python3-dev && \
29+
pip3 install -U crcmod && \
30+
apt-get autoremove -y gcc python3-dev
2431

32+
33+
ENV PATH=/opt/gcloud/google-cloud-sdk/bin/:$PATH
2534
ENV OSS_FUZZ_ROOT=/opt/oss-fuzz
2635
ADD . ${OSS_FUZZ_ROOT}
2736
RUN python3 -m pip install -r ${OSS_FUZZ_ROOT}/infra/cifuzz/requirements.txt

‎infra/cifuzz/cifuzz_end_to_end_test.py

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ def test_simple(self):
3939
"""Simple end-to-end test using run_cifuzz.main()."""
4040
os.environ['REPOSITORY'] = 'external-project'
4141
os.environ['PROJECT_SRC_PATH'] = EXTERNAL_PROJECT_PATH
42+
os.environ['FILESTORE'] = 'no_filestore'
43+
os.environ['NO_CLUSTERFUZZ_DEPLOYMENT'] = 'True'
4244

4345
with test_helpers.docker_temp_dir() as temp_dir:
4446
os.environ['WORKSPACE'] = temp_dir

‎infra/cifuzz/clusterfuzz_deployment.py

+8-9
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ def download_latest_build(self):
100100
# called if multiple bugs are found.
101101
return self.workspace.clusterfuzz_build
102102

103-
repo_dir = self.ci_system.repo_dir()
103+
repo_dir = self.ci_system.repo_dir
104104
if not repo_dir:
105105
raise RuntimeError('Repo checkout does not exist.')
106106

@@ -355,20 +355,19 @@ def get_coverage(self, repo_path):
355355

356356

357357
_PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING = {
358-
config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI:
359-
OSSFuzz,
360-
config_utils.BaseConfig.Platform.INTERNAL_GITHUB:
361-
OSSFuzz,
362-
config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI:
363-
NoClusterFuzzDeployment,
364-
config_utils.BaseConfig.Platform.EXTERNAL_GITHUB:
365-
ClusterFuzzLite,
358+
config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI: OSSFuzz,
359+
config_utils.BaseConfig.Platform.INTERNAL_GITHUB: OSSFuzz,
360+
config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI: ClusterFuzzLite,
361+
config_utils.BaseConfig.Platform.EXTERNAL_GITHUB: ClusterFuzzLite,
366362
}
367363

368364

369365
def get_clusterfuzz_deployment(config, workspace):
370366
"""Returns object reprsenting deployment of ClusterFuzz used by |config|."""
371367
deployment_cls = _PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING[config.platform]
368+
if config.no_clusterfuzz_deployment:
369+
logging.info('Overriding ClusterFuzzDeployment. Using None.')
370+
deployment_cls = NoClusterFuzzDeployment
372371
result = deployment_cls(config, workspace)
373372
logging.info('ClusterFuzzDeployment: %s.', result)
374373
return result

‎infra/cifuzz/clusterfuzz_deployment_test.py

+8-4
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ def setUp(self):
132132
self.setUpPyfakefs()
133133
self.deployment = _create_deployment(run_fuzzers_mode='batch',
134134
oss_fuzz_project_name='',
135+
cloud_bucket='gs://bucket',
135136
is_github=True)
136137
self.corpus_dir = os.path.join(self.deployment.workspace.corpora,
137138
EXAMPLE_FUZZER)
@@ -157,7 +158,7 @@ def test_download_corpus_fail(self, _):
157158
side_effect=[False, True])
158159
@mock.patch('repo_manager.RepoManager.get_commit_list',
159160
return_value=['commit1', 'commit2'])
160-
@mock.patch('continuous_integration.BaseCi.repo_dir',
161+
@mock.patch('continuous_integration.GithubCiMixin.repo_dir',
161162
return_value='/path/to/repo')
162163
def test_download_latest_build(self, mock_repo_dir, mock_get_commit_list,
163164
mock_download_build):
@@ -173,7 +174,7 @@ def test_download_latest_build(self, mock_repo_dir, mock_get_commit_list,
173174
side_effect=Exception)
174175
@mock.patch('repo_manager.RepoManager.get_commit_list',
175176
return_value=['commit1', 'commit2'])
176-
@mock.patch('continuous_integration.BaseCi.repo_dir',
177+
@mock.patch('continuous_integration.GithubCiMixin.repo_dir',
177178
return_value='/path/to/repo')
178179
def test_download_latest_build_fail(self, mock_repo_dir, mock_get_commit_list,
179180
_):
@@ -195,10 +196,13 @@ class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase):
195196
def setUp(self):
196197
self.setUpPyfakefs()
197198
config = test_helpers.create_run_config(workspace=WORKSPACE,
198-
is_github=False)
199+
is_github=False,
200+
filestore='no_filestore',
201+
no_clusterfuzz_deployment=True)
199202
workspace = workspace_utils.Workspace(config)
200203
self.deployment = clusterfuzz_deployment.get_clusterfuzz_deployment(
201204
config, workspace)
205+
202206
self.corpus_dir = os.path.join(workspace.corpora, EXAMPLE_FUZZER)
203207

204208
@mock.patch('logging.info')
@@ -241,7 +245,7 @@ def setUp(self):
241245
(config_utils.BaseConfig.Platform.INTERNAL_GITHUB,
242246
clusterfuzz_deployment.OSSFuzz),
243247
(config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI,
244-
clusterfuzz_deployment.NoClusterFuzzDeployment),
248+
clusterfuzz_deployment.ClusterFuzzLite),
245249
(config_utils.BaseConfig.Platform.EXTERNAL_GITHUB,
246250
clusterfuzz_deployment.ClusterFuzzLite),
247251
])

‎infra/cifuzz/config_utils.py

+9-1
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,12 @@ def __init__(self):
236236
self.git_store_branch = os.environ.get('GIT_STORE_BRANCH')
237237
self.git_store_branch_coverage = os.environ.get('GIT_STORE_BRANCH_COVERAGE',
238238
self.git_store_branch)
239+
self.project_src_path = self._ci_env.project_src_path
239240
self.docker_in_docker = os.environ.get('DOCKER_IN_DOCKER')
241+
self.filestore = os.environ.get('FILESTORE')
242+
self.cloud_bucket = os.environ.get('CLOUD_BUCKET')
243+
self.no_clusterfuzz_deployment = os.environ.get('NO_CLUSTERFUZZ_DEPLOYMENT',
244+
False)
240245

241246
# TODO(metzman): Fix tests to create valid configurations and get rid of
242247
# CIFUZZ_TEST here and in presubmit.py.
@@ -261,6 +266,10 @@ def validate(self):
261266
constants.LANGUAGES)
262267
return False
263268

269+
if not self.project_repo_name:
270+
logging.error('Must set REPOSITORY.')
271+
return False
272+
264273
return True
265274

266275
@property
@@ -361,7 +370,6 @@ def __init__(self):
361370
self._get_config_from_event_path(event)
362371

363372
self.base_ref = os.getenv('GITHUB_BASE_REF')
364-
self.project_src_path = self._ci_env.project_src_path
365373

366374
self.allowed_broken_targets_percentage = os.getenv(
367375
'ALLOWED_BROKEN_TARGETS_PERCENTAGE')

‎infra/cifuzz/config_utils_test.py

+1
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ def test_validate(self):
9191
"""Tests that validate returns True if config is valid."""
9292
os.environ['OSS_FUZZ_PROJECT_NAME'] = 'example'
9393
os.environ['WORKSPACE'] = '/workspace'
94+
os.environ['REPOSITORY'] = 'repo'
9495
config = self._create_config()
9596
self.assertTrue(config.validate())
9697

‎infra/cifuzz/continuous_integration.py

+48-13
Original file line numberDiff line numberDiff line change
@@ -53,23 +53,13 @@ class BaseCi:
5353
def __init__(self, config):
5454
self.config = config
5555
self.workspace = workspace_utils.Workspace(config)
56+
self._repo_dir = None
5657

58+
@property
5759
def repo_dir(self):
5860
"""Returns the source repo path, if it has been checked out. None is
5961
returned otherwise."""
60-
if not os.path.exists(self.workspace.repo_storage):
61-
return None
62-
63-
# Note: this assumes there is only one repo checked out here.
64-
listing = os.listdir(self.workspace.repo_storage)
65-
if len(listing) != 1:
66-
raise RuntimeError('Invalid repo storage.')
67-
68-
repo_path = os.path.join(self.workspace.repo_storage, listing[0])
69-
if not os.path.isdir(repo_path):
70-
raise RuntimeError('Repo is not a directory.')
71-
72-
return repo_path
62+
raise NotImplementedError('Child class must implement method.')
7363

7464
def prepare_for_fuzzer_build(self):
7565
"""Builds the fuzzer builder image and gets the source code we need to
@@ -152,6 +142,31 @@ def checkout_specified_commit(repo_manager_obj, pr_ref, commit_sha):
152142
class GithubCiMixin:
153143
"""Mixin for Github based CI systems."""
154144

145+
def __init__(self, config):
146+
super().__init__(config)
147+
# Unlike in other classes, here _repo_dir is the parent directory of the
148+
# repo, not its actual directory.
149+
self._repo_dir = self.workspace.repo_storage
150+
151+
@property
152+
def repo_dir(self):
153+
"""Returns the source repo path, if it has been checked out. None is
154+
returned otherwise."""
155+
if not os.path.exists(self._repo_dir):
156+
logging.warning('Repo dir: %s does not exist.', self._repo_dir)
157+
return None
158+
159+
# Note: this assumes there is only one repo checked out here.
160+
listing = os.listdir(self._repo_dir)
161+
if len(listing) != 1:
162+
raise RuntimeError('Invalid repo directory.')
163+
164+
repo_path = os.path.join(self._repo_dir, listing[0])
165+
if not os.path.isdir(repo_path):
166+
raise RuntimeError('Repo is not a directory.')
167+
168+
return repo_path
169+
155170
def get_diff_base(self):
156171
"""Returns the base to diff against with git to get the change under
157172
test."""
@@ -217,6 +232,16 @@ class InternalGeneric(BaseCi):
217232
"""Class representing CI for an OSS-Fuzz project on a CI other than Github
218233
actions."""
219234

235+
def __init__(self, config):
236+
super().__init__(config)
237+
self._repo_dir = config.project_src_path
238+
239+
@property
240+
def repo_dir(self):
241+
"""Returns the source repo path, if it has been checked out. None is
242+
returned otherwise."""
243+
return self._repo_dir
244+
220245
def prepare_for_fuzzer_build(self):
221246
"""Builds the project builder image for an OSS-Fuzz project outside of
222247
GitHub actions. Returns the repo_manager. Does not checkout source code
@@ -263,6 +288,16 @@ def build_external_project_docker_image(project_src, build_integration_path):
263288
class ExternalGeneric(BaseCi):
264289
"""CI implementation for generic CI for external (non-OSS-Fuzz) projects."""
265290

291+
def __init__(self, config):
292+
super().__init__(config)
293+
self._repo_dir = config.project_src_path
294+
295+
@property
296+
def repo_dir(self):
297+
"""Returns the source repo path, if it has been checked out. None is
298+
returned otherwise."""
299+
return self._repo_dir
300+
266301
def get_diff_base(self):
267302
return 'origin...'
268303

‎infra/cifuzz/filestore/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,6 @@ def download_build(self, name, dst_directory):
4949
"""Downloads the build with |name| to |dst_directory|."""
5050
raise NotImplementedError('Child class must implement method.')
5151

52-
def download_coverage(self, dst_directory):
52+
def download_coverage(self, name, dst_directory):
5353
"""Downloads the latest project coverage report."""
5454
raise NotImplementedError('Child class must implement method.')

‎infra/cifuzz/filestore/github_actions/__init__.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@
2121

2222
# pylint: disable=wrong-import-position,import-error
2323
sys.path.append(
24-
os.path.join(os.path.pardir, os.path.pardir, os.path.pardir,
25-
os.path.dirname(os.path.abspath(__file__))))
24+
os.path.abspath(
25+
os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir,
26+
os.path.pardir)))
2627

2728
import utils
2829
import http_utils

‎infra/cifuzz/filestore/github_actions/github_actions_test.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@
2424

2525
# pylint: disable=wrong-import-position
2626
INFRA_DIR = os.path.dirname(
27-
os.path.dirname(os.path.dirname(os.path.dirname(
28-
os.path.abspath(__file__)))))
27+
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
2928
sys.path.append(INFRA_DIR)
3029

3130
from filestore import github_actions

‎infra/cifuzz/filestore/github_actions/github_api_test.py

+8
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,16 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414
"""Tests for github_api."""
15+
import os
16+
import sys
1517
import unittest
1618

19+
# pylint: disable=wrong-import-position,import-error
20+
sys.path.append(
21+
os.path.abspath(
22+
os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir,
23+
os.path.pardir)))
24+
1725
from filestore.github_actions import github_api
1826
import test_helpers
1927

0 commit comments

Comments
 (0)