Skip to content

Commit 946e5b5

Browse files
davidsgrogandavidsgrogan
committed
Update to leveldb 1.6
Highlights ---------- Mmap at most 1000 files on Posix to improve performance for large databases. Support for more architectures (thanks to Alexander K.) Building and porting -------------------- HP/UX support (issue 126) AtomicPointer for ia64 (issue 123) Sparc v9 support (issue 124) Atomic ops for powerpc Use -fno-builtin-memcmp only when using g++ Simplify IOS build rules (issue 114) Use CXXFLAGS instead of CFLAGS when invoking C++ compiler (issue 118) Fix snappy shared library problem (issue 94) Fix shared library installation path regression Endian-ness detection tweak for FreeBSD Bug fixes --------- Stop ignoring FLAGS_open_files in db_bench Make bloom test behavior agnostic to endian-ness Performance ----------- Limit number of mmapped files to 1000 to improve perf for large dbs Do not delay for 1 second on shutdown path (issue 125) Misc ---- Make InMemoryEnv return a no-op logger C binding now has a wrapper for free (issue 117) Add thread-safety annotations Added an in-process lock table (issue 120) Make RandomAccessFile and SequentialFile non-copyable
1 parent dd0d562 commit 946e5b5

25 files changed

+382
-72
lines changed

Diff for: ‎Makefile

+6-5
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode)
1515
#-----------------------------------------------
1616

1717
# detect what platform we're building on
18-
$(shell ./build_detect_platform build_config.mk)
18+
$(shell CC=$(CC) CXX=$(CXX) TARGET_OS=$(TARGET_OS) \
19+
./build_detect_platform build_config.mk ./)
1920
# this file is generated by the previous line to set build flags and sources
2021
include build_config.mk
2122

@@ -70,7 +71,7 @@ SHARED = $(SHARED1)
7071
else
7172
# Update db.h if you change these.
7273
SHARED_MAJOR = 1
73-
SHARED_MINOR = 5
74+
SHARED_MINOR = 6
7475
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
7576
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
7677
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@@ -82,7 +83,7 @@ $(SHARED2): $(SHARED3)
8283
endif
8384

8485
$(SHARED3):
85-
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
86+
$(CXX) $(SOURCES) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -o $(SHARED3)
8687

8788
endif # PLATFORM_SHARED_EXT
8889

@@ -179,14 +180,14 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBu
179180

180181
.cc.o:
181182
mkdir -p ios-x86/$(dir $@)
182-
$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
183+
$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
183184
mkdir -p ios-arm/$(dir $@)
184185
$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
185186
lipo ios-x86/$@ ios-arm/$@ -create -output $@
186187

187188
.c.o:
188189
mkdir -p ios-x86/$(dir $@)
189-
$(SIMULATORROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
190+
$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
190191
mkdir -p ios-arm/$(dir $@)
191192
$(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
192193
lipo ios-x86/$@ ios-arm/$@ -create -output $@

Diff for: ‎TODO

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ db
77
within [start_key..end_key]? For Chrome, deletion of obsolete
88
object stores, etc. can be done in the background anyway, so
99
probably not that important.
10+
- There have been requests for MultiGet.
1011

1112
After a range is completely deleted, what gets rid of the
1213
corresponding files if we do no future changes to that range. Make

Diff for: ‎build_detect_platform

+33-16
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@
2323
#
2424

2525
OUTPUT=$1
26-
if test -z "$OUTPUT"; then
27-
echo "usage: $0 <output-filename>" >&2
26+
PREFIX=$2
27+
if test -z "$OUTPUT" || test -z "$PREFIX"; then
28+
echo "usage: $0 <output-filename> <directory_prefix>" >&2
2829
exit 1
2930
fi
3031

@@ -55,58 +56,72 @@ PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
5556
PLATFORM_SHARED_CFLAGS="-fPIC"
5657
PLATFORM_SHARED_VERSIONED=true
5758

58-
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
59+
MEMCMP_FLAG=
60+
if [ "$CXX" = "g++" ]; then
61+
# Use libc's memcmp instead of GCC's memcmp. This results in ~40%
62+
# performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
63+
MEMCMP_FLAG="-fno-builtin-memcmp"
64+
fi
65+
5966
case "$TARGET_OS" in
6067
Darwin)
6168
PLATFORM=OS_MACOSX
62-
COMMON_FLAGS="-fno-builtin-memcmp -DOS_MACOSX"
69+
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
6370
PLATFORM_SHARED_EXT=dylib
6471
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
6572
PORT_FILE=port/port_posix.cc
6673
;;
6774
Linux)
6875
PLATFORM=OS_LINUX
69-
COMMON_FLAGS="-fno-builtin-memcmp -pthread -DOS_LINUX"
76+
COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
7077
PLATFORM_LDFLAGS="-pthread"
7178
PORT_FILE=port/port_posix.cc
7279
;;
7380
SunOS)
7481
PLATFORM=OS_SOLARIS
75-
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
82+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
7683
PLATFORM_LDFLAGS="-lpthread -lrt"
7784
PORT_FILE=port/port_posix.cc
7885
;;
7986
FreeBSD)
8087
PLATFORM=OS_FREEBSD
81-
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
88+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
8289
PLATFORM_LDFLAGS="-lpthread"
8390
PORT_FILE=port/port_posix.cc
8491
;;
8592
NetBSD)
8693
PLATFORM=OS_NETBSD
87-
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
94+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
8895
PLATFORM_LDFLAGS="-lpthread -lgcc_s"
8996
PORT_FILE=port/port_posix.cc
9097
;;
9198
OpenBSD)
9299
PLATFORM=OS_OPENBSD
93-
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
100+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
94101
PLATFORM_LDFLAGS="-pthread"
95102
PORT_FILE=port/port_posix.cc
96103
;;
97104
DragonFly)
98105
PLATFORM=OS_DRAGONFLYBSD
99-
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
106+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
100107
PLATFORM_LDFLAGS="-lpthread"
101108
PORT_FILE=port/port_posix.cc
102109
;;
103110
OS_ANDROID_CROSSCOMPILE)
104111
PLATFORM=OS_ANDROID
105-
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
112+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
106113
PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
107114
PORT_FILE=port/port_posix.cc
108115
CROSS_COMPILE=true
109116
;;
117+
HP-UX)
118+
PLATFORM=OS_HPUX
119+
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
120+
PLATFORM_LDFLAGS="-pthread"
121+
PORT_FILE=port/port_posix.cc
122+
# man ld: +h internal_name
123+
PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
124+
;;
110125
*)
111126
echo "Unknown platform!" >&2
112127
exit 1
@@ -116,11 +131,13 @@ esac
116131
# except for the test and benchmark files. By default, find will output a list
117132
# of all files matching either rule, so we need to append -print to make the
118133
# prune take effect.
119-
DIRS="util db table"
134+
DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
135+
120136
set -f # temporarily disable globbing so that our patterns aren't expanded
121137
PRUNE_TEST="-name *test*.cc -prune"
122138
PRUNE_BENCH="-name *_bench.cc -prune"
123-
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
139+
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
140+
124141
set +f # re-enable globbing
125142

126143
# The sources consist of the portable files, plus the platform-specific port
@@ -133,7 +150,7 @@ if [ "$CROSS_COMPILE" = "true" ]; then
133150
true
134151
else
135152
# If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h.
136-
$CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
153+
$CXX $CXXFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
137154
#include <cstdatomic>
138155
int main() {}
139156
EOF
@@ -146,7 +163,7 @@ EOF
146163

147164
# Test whether Snappy library is installed
148165
# http://code.google.com/p/snappy/
149-
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
166+
$CXX $CXXFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
150167
#include <snappy.h>
151168
int main() {}
152169
EOF
@@ -156,7 +173,7 @@ EOF
156173
fi
157174

158175
# Test whether tcmalloc is available
159-
$CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
176+
$CXX $CXXFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
160177
int main() {}
161178
EOF
162179
if [ "$?" = 0 ]; then

Diff for: ‎db/c.cc

+4
Original file line numberDiff line numberDiff line change
@@ -578,4 +578,8 @@ void leveldb_env_destroy(leveldb_env_t* env) {
578578
delete env;
579579
}
580580

581+
void leveldb_free(void* ptr) {
582+
free(ptr);
583+
}
584+
581585
} // end extern "C"

Diff for: ‎db/c_test.c

+6
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,12 @@ int main(int argc, char** argv) {
204204
CheckCondition(err != NULL);
205205
Free(&err);
206206

207+
StartPhase("leveldb_free");
208+
db = leveldb_open(options, dbname, &err);
209+
CheckCondition(err != NULL);
210+
leveldb_free(err);
211+
err = NULL;
212+
207213
StartPhase("open");
208214
leveldb_options_set_create_if_missing(options, 1);
209215
db = leveldb_open(options, dbname, &err);

Diff for: ‎db/db_bench.cc

+1
Original file line numberDiff line numberDiff line change
@@ -693,6 +693,7 @@ class Benchmark {
693693
options.create_if_missing = !FLAGS_use_existing_db;
694694
options.block_cache = cache_;
695695
options.write_buffer_size = FLAGS_write_buffer_size;
696+
options.max_open_files = FLAGS_open_files;
696697
options.filter_policy = filter_policy_;
697698
Status s = DB::Open(options, FLAGS_db, &db_);
698699
if (!s.ok()) {

Diff for: ‎db/db_impl.cc

+5-1
Original file line numberDiff line numberDiff line change
@@ -609,7 +609,11 @@ void DBImpl::BackgroundCall() {
609609
assert(bg_compaction_scheduled_);
610610
if (!shutting_down_.Acquire_Load()) {
611611
Status s = BackgroundCompaction();
612-
if (!s.ok()) {
612+
if (s.ok()) {
613+
// Success
614+
} else if (shutting_down_.Acquire_Load()) {
615+
// Error most likely due to shutdown; do not wait
616+
} else {
613617
// Wait a little bit before retrying background compaction in
614618
// case this is an environmental problem and we do not want to
615619
// chew up resources for failed compactions for the duration of

Diff for: ‎db/db_impl.h

+18-10
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "leveldb/db.h"
1414
#include "leveldb/env.h"
1515
#include "port/port.h"
16+
#include "port/thread_annotations.h"
1617

1718
namespace leveldb {
1819

@@ -71,7 +72,7 @@ class DBImpl : public DB {
7172
// Recover the descriptor from persistent storage. May do a significant
7273
// amount of work to recover recently logged updates. Any changes to
7374
// be made to the descriptor are added to *edit.
74-
Status Recover(VersionEdit* edit);
75+
Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_);
7576

7677
void MaybeIgnoreError(Status* s) const;
7778

@@ -80,27 +81,34 @@ class DBImpl : public DB {
8081

8182
// Compact the in-memory write buffer to disk. Switches to a new
8283
// log-file/memtable and writes a new descriptor iff successful.
83-
Status CompactMemTable();
84+
Status CompactMemTable()
85+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
8486

8587
Status RecoverLogFile(uint64_t log_number,
8688
VersionEdit* edit,
87-
SequenceNumber* max_sequence);
89+
SequenceNumber* max_sequence)
90+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
8891

89-
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base);
92+
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
93+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
9094

91-
Status MakeRoomForWrite(bool force /* compact even if there is room? */);
95+
Status MakeRoomForWrite(bool force /* compact even if there is room? */)
96+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
9297
WriteBatch* BuildBatchGroup(Writer** last_writer);
9398

94-
void MaybeScheduleCompaction();
99+
void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
95100
static void BGWork(void* db);
96101
void BackgroundCall();
97-
Status BackgroundCompaction();
98-
void CleanupCompaction(CompactionState* compact);
99-
Status DoCompactionWork(CompactionState* compact);
102+
Status BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
103+
void CleanupCompaction(CompactionState* compact)
104+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
105+
Status DoCompactionWork(CompactionState* compact)
106+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
100107

101108
Status OpenCompactionOutputFile(CompactionState* compact);
102109
Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
103-
Status InstallCompactionResults(CompactionState* compact);
110+
Status InstallCompactionResults(CompactionState* compact)
111+
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
104112

105113
// Constant after construction
106114
Env* const env_;

Diff for: ‎db/db_test.cc

+6
Original file line numberDiff line numberDiff line change
@@ -1442,6 +1442,12 @@ TEST(DBTest, DBOpen_Options) {
14421442
db = NULL;
14431443
}
14441444

1445+
TEST(DBTest, Locking) {
1446+
DB* db2 = NULL;
1447+
Status s = DB::Open(CurrentOptions(), dbname_, &db2);
1448+
ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
1449+
}
1450+
14451451
// Check that number of files does not grow when we are out of space
14461452
TEST(DBTest, NoSpace) {
14471453
Options options = CurrentOptions();

Diff for: ‎db/version_set.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -865,7 +865,7 @@ Status VersionSet::Recover() {
865865
if (edit.has_comparator_ &&
866866
edit.comparator_ != icmp_.user_comparator()->Name()) {
867867
s = Status::InvalidArgument(
868-
edit.comparator_ + "does not match existing comparator ",
868+
edit.comparator_ + " does not match existing comparator ",
869869
icmp_.user_comparator()->Name());
870870
}
871871
}

Diff for: ‎db/version_set.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "db/dbformat.h"
2222
#include "db/version_edit.h"
2323
#include "port/port.h"
24+
#include "port/thread_annotations.h"
2425

2526
namespace leveldb {
2627

@@ -159,7 +160,8 @@ class VersionSet {
159160
// current version. Will release *mu while actually writing to the file.
160161
// REQUIRES: *mu is held on entry.
161162
// REQUIRES: no other thread concurrently calls LogAndApply()
162-
Status LogAndApply(VersionEdit* edit, port::Mutex* mu);
163+
Status LogAndApply(VersionEdit* edit, port::Mutex* mu)
164+
EXCLUSIVE_LOCKS_REQUIRED(mu);
163165

164166
// Recover the last saved descriptor from persistent storage.
165167
Status Recover();

Diff for: ‎doc/bench/db_bench_sqlite3.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ class Benchmark {
618618
ErrorCheck(status);
619619

620620
// Execute read statement
621-
while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW);
621+
while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {}
622622
StepErrorCheck(status);
623623

624624
// Reset SQLite statement for another use

Diff for: ‎doc/index.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ <h2>Filters</h2>
408408
the number of disk reads substantially.
409409
<pre>
410410
leveldb::Options options;
411-
options.filter_policy = NewBloomFilter(10);
411+
options.filter_policy = NewBloomFilterPolicy(10);
412412
leveldb::DB* db;
413413
leveldb::DB::Open(options, "/tmp/testdb", &amp;db);
414414
... use the database ...

Diff for: ‎helpers/memenv/memenv.cc

+10
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,11 @@ class WritableFileImpl : public WritableFile {
221221
FileState* file_;
222222
};
223223

224+
class NoOpLogger : public Logger {
225+
public:
226+
virtual void Logv(const char* format, va_list ap) { }
227+
};
228+
224229
class InMemoryEnv : public EnvWrapper {
225230
public:
226231
explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
@@ -358,6 +363,11 @@ class InMemoryEnv : public EnvWrapper {
358363
return Status::OK();
359364
}
360365

366+
virtual Status NewLogger(const std::string& fname, Logger** result) {
367+
*result = new NoOpLogger;
368+
return Status::OK();
369+
}
370+
361371
private:
362372
// Map from filenames to FileState objects, representing a simple file system.
363373
typedef std::map<std::string, FileState*> FileSystem;

Diff for: ‎include/leveldb/c.h

+10
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
be true on entry:
2929
*errptr == NULL
3030
*errptr points to a malloc()ed null-terminated error message
31+
(On Windows, *errptr must have been malloc()-ed by this library.)
3132
On success, a leveldb routine leaves *errptr unchanged.
3233
On failure, leveldb frees the old value of *errptr and
3334
set *errptr to a malloc()ed error message.
@@ -268,6 +269,15 @@ extern void leveldb_cache_destroy(leveldb_cache_t* cache);
268269
extern leveldb_env_t* leveldb_create_default_env();
269270
extern void leveldb_env_destroy(leveldb_env_t*);
270271

272+
/* Utility */
273+
274+
/* Calls free(ptr).
275+
REQUIRES: ptr was malloc()-ed and returned by one of the routines
276+
in this file. Note that in certain cases (typically on Windows), you
277+
may need to call this routine instead of free(ptr) to dispose of
278+
malloc()-ed memory returned by this library. */
279+
extern void leveldb_free(void* ptr);
280+
271281
#ifdef __cplusplus
272282
} /* end extern "C" */
273283
#endif

0 commit comments

Comments
 (0)