From 5fcdcf95cb62ab3d593c36ef90df27cef63874a1 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 31 Dec 2016 03:12:51 -0500 Subject: [PATCH 1/9] Make TimeStamp 64-bit. This prepares it for higher-resolution timestamps. --- src/build_log.cc | 2 +- src/deps_log.cc | 13 ++++++------- src/deps_log.h | 7 ++++--- src/graph.cc | 8 ++++---- src/ninja.cc | 2 +- src/timestamp.h | 6 +++--- src/util.h | 1 + src/win32port.h | 1 + 8 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/build_log.cc b/src/build_log.cc index 333915af9f..a591050d8f 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -353,7 +353,7 @@ BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { } bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { - return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", + return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n", entry.start_time, entry.end_time, entry.mtime, entry.output.c_str(), entry.command_hash) > 0; } diff --git a/src/deps_log.cc b/src/deps_log.cc index 89c60232b7..d7f0b26737 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -30,7 +30,7 @@ // The version is stored as 4 bytes after the signature and also serves as a // byte order mark. Signature and version combined are 16 bytes long. const char kFileSignature[] = "# ninjadeps\n"; -const int kCurrentVersion = 3; +const int kCurrentVersion = 4; // Record size is currently limited to less than the full 32 bit, due to // internal buffers having to have this size. @@ -124,7 +124,7 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, return true; // Update on-disk representation. - unsigned size = 4 * (1 + 1 + node_count); + unsigned size = 4 * (1 + 2 + node_count); if (size > kMaxRecordSize) { errno = ERANGE; return false; @@ -135,8 +135,7 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, int id = node->id(); if (fwrite(&id, 4, 1, file_) < 1) return false; - int timestamp = mtime; - if (fwrite(×tamp, 4, 1, file_) < 1) + if (fwrite(&mtime, 8, 1, file_) < 1) return false; for (int i = 0; i < node_count; ++i) { id = nodes[i]->id(); @@ -218,9 +217,9 @@ bool DepsLog::Load(const string& path, State* state, string* err) { assert(size % 4 == 0); int* deps_data = reinterpret_cast(buf); int out_id = deps_data[0]; - int mtime = deps_data[1]; - deps_data += 2; - int deps_count = (size / 4) - 2; + TimeStamp mtime = reinterpret_cast(&deps_data[1])[0]; + deps_data += 3; + int deps_count = (size / 4) - 3; Deps* deps = new Deps(mtime, deps_count); for (int i = 0; i < deps_count; ++i) { diff --git a/src/deps_log.h b/src/deps_log.h index cec0257cef..793af07910 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -57,7 +57,8 @@ struct State; /// one's complement of the expected index of the record (to detect /// concurrent writes of multiple ninja processes to the log). /// dependency records are an array of 4-byte integers -/// [output path id, output path mtime, input path id, input path id...] +/// [output path id, output path mtime (8-byte int), input path id, +/// input path id...] /// (The mtime is compared against the on-disk output path mtime /// to verify the stored data is up-to-date.) /// If two records reference the same output the latter one in the file @@ -75,10 +76,10 @@ struct DepsLog { // Reading (startup-time) interface. struct Deps { - Deps(int mtime, int node_count) + Deps(int64_t mtime, int node_count) : mtime(mtime), node_count(node_count), nodes(new Node*[node_count]) {} ~Deps() { delete [] nodes; } - int mtime; + TimeStamp mtime; int node_count; Node** nodes; }; diff --git a/src/graph.cc b/src/graph.cc index ce4ea774f2..b41c247912 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -233,7 +233,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, if (output_mtime < most_recent_input->mtime()) { EXPLAIN("%soutput %s older than most recent input %s " - "(%d vs %d)", + "(%" PRId64 " vs %" PRId64 ")", used_restat ? "restat of " : "", output->path().c_str(), most_recent_input->path().c_str(), output_mtime, most_recent_input->mtime()); @@ -257,7 +257,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, // mtime of the most recent input. This can occur even when the mtime // on disk is newer if a previous run wrote to the output file but // exited with an error or was interrupted. - EXPLAIN("recorded mtime of %s older than most recent input %s (%d vs %d)", + EXPLAIN("recorded mtime of %s older than most recent input %s (%" PRId64 " vs %" PRId64 ")", output->path().c_str(), most_recent_input->path().c_str(), entry->mtime, most_recent_input->mtime()); return true; @@ -441,7 +441,7 @@ string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { } void Node::Dump(const char* prefix) const { - printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", + printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ", prefix, path().c_str(), this, mtime(), mtime() ? "" : " (:missing)", dirty() ? " dirty" : " clean"); @@ -547,7 +547,7 @@ bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, string* err) { // Deps are invalid if the output is newer than the deps. if (output->mtime() > deps->mtime) { - EXPLAIN("stored deps info out of date for '%s' (%d vs %d)", + EXPLAIN("stored deps info out of date for '%s' (%" PRId64 " vs %" PRId64 ")", output->path().c_str(), deps->mtime, output->mtime()); return false; } diff --git a/src/ninja.cc b/src/ninja.cc index ed004ac8f1..a9030aa0f5 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -494,7 +494,7 @@ int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) { TimeStamp mtime = disk_interface.Stat((*it)->path(), &err); if (mtime == -1) Error("%s", err.c_str()); // Log and ignore Stat() errors; - printf("%s: #deps %d, deps mtime %d (%s)\n", + printf("%s: #deps %d, deps mtime %" PRId64 " (%s)\n", (*it)->path().c_str(), deps->node_count, deps->mtime, (!mtime || mtime > deps->mtime ? "STALE":"VALID")); for (int i = 0; i < deps->node_count; ++i) diff --git a/src/timestamp.h b/src/timestamp.h index cee7ba8f21..58ae148c2f 100644 --- a/src/timestamp.h +++ b/src/timestamp.h @@ -17,8 +17,8 @@ // When considering file modification times we only care to compare // them against one another -- we never convert them to an absolute -// real time. On POSIX we use time_t (seconds since epoch) and on -// Windows we use a different value. Both fit in an int. -typedef int TimeStamp; +// real time. On POSIX we use timespec (seconds&nanoseconds since epoch) +// and on Windows we use a different value. Both fit in an int64. +typedef int64_t TimeStamp; #endif // NINJA_TIMESTAMP_H_ diff --git a/src/util.h b/src/util.h index 4ee41a500a..a8493d69f4 100644 --- a/src/util.h +++ b/src/util.h @@ -18,6 +18,7 @@ #ifdef _WIN32 #include "win32port.h" #else +#include #include #endif diff --git a/src/win32port.h b/src/win32port.h index ce3c9498e5..546479762b 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -23,6 +23,7 @@ typedef unsigned long long uint64_t; // printf format specifier for uint64_t, from C99. #ifndef PRIu64 +#define PRId64 "I64d" #define PRIu64 "I64u" #define PRIx64 "I64x" #endif From f50a5250e49347df802d7844358f5ea567ddfac2 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 31 Dec 2016 03:33:01 -0500 Subject: [PATCH 2/9] Read file timestamps in higher resolution. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This uses nanoseconds on POSIX (±~292 years) and 100-ns increments on Windows (±~29247 years). The fallbacks to different structure fields is the only thing grabbed from #337, with a slight modification in implementation. --- src/disk_interface.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 28530b19d0..ffa58db94d 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -61,12 +61,11 @@ int MakeDir(const string& path) { TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { // FILETIME is in 100-nanosecond increments since the Windows epoch. // We don't much care about epoch correctness but we do want the - // resulting value to fit in an integer. + // resulting value to fit in a 64-bit integer. uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | ((uint64_t)filetime.dwLowDateTime); - mtime /= 1000000000LL / 100; // 100ns -> s. - mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime; + // 1600 epoch -> 2000 epoch (subtract 400 years). + return (TimeStamp)mtime - 12622770400LL * 1000000000LL / 100; } TimeStamp StatSingleFile(const string& path, string* err) { @@ -192,7 +191,17 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { // that it doesn't exist. if (st.st_mtime == 0) return 1; - return st.st_mtime; +#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE) + return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL + + st.st_mtimespec.tv_nsec); +#elif defined(_LARGEFILE64_SOURCE) + return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec; +#elif defined(__CYGWIN__) + return (int64_t)st.st_mtime * 1000000000LL; +#else + // see http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html + return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec; +#endif #endif } From 9fa250d794cabc68fe53087de1b5b9797e515e18 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 1 Jan 2017 03:51:09 -0500 Subject: [PATCH 3/9] Move #include for type definition to correct place. Not sure why the old way works in newer compilers; maybe they just pre-define these types by defaulting to a newer standard. --- src/timestamp.h | 6 ++++++ src/util.h | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/timestamp.h b/src/timestamp.h index 58ae148c2f..556515433b 100644 --- a/src/timestamp.h +++ b/src/timestamp.h @@ -15,6 +15,12 @@ #ifndef NINJA_TIMESTAMP_H_ #define NINJA_TIMESTAMP_H_ +#ifdef _WIN32 +#include "win32port.h" +#else +#include +#endif + // When considering file modification times we only care to compare // them against one another -- we never convert them to an absolute // real time. On POSIX we use timespec (seconds&nanoseconds since epoch) diff --git a/src/util.h b/src/util.h index a8493d69f4..4ee41a500a 100644 --- a/src/util.h +++ b/src/util.h @@ -18,7 +18,6 @@ #ifdef _WIN32 #include "win32port.h" #else -#include #include #endif From eb6cea27f23c0674cea47871f281aaffb8a8efc3 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 1 Jan 2017 04:06:56 -0500 Subject: [PATCH 4/9] Add #define to get printf-format specifiers. This is needed on older compilers/stdlibs such as on Ubuntu Precise which is used on Travis. --- src/timestamp.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/timestamp.h b/src/timestamp.h index 556515433b..6a7ccd0b06 100644 --- a/src/timestamp.h +++ b/src/timestamp.h @@ -18,6 +18,9 @@ #ifdef _WIN32 #include "win32port.h" #else +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif #include #endif From bf4fb03e4b266e7f75f88664b6a941d20650046b Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 20 Apr 2017 23:10:56 -0400 Subject: [PATCH 5/9] Use 64-bit-alignment-safe timestamp reading. Read and write the timestamp as two separate 32-bit integers in a fixed order to prevent any issues with alignment or byte order. --- src/deps_log.cc | 10 ++++++++-- src/deps_log.h | 5 +++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/deps_log.cc b/src/deps_log.cc index d7f0b26737..42394265d8 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -135,7 +135,11 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, int id = node->id(); if (fwrite(&id, 4, 1, file_) < 1) return false; - if (fwrite(&mtime, 8, 1, file_) < 1) + uint32_t mtime_part = static_cast(mtime & 0xffffffff); + if (fwrite(&mtime_part, 4, 1, file_) < 1) + return false; + mtime_part = static_cast((mtime >> 32) & 0xffffffff); + if (fwrite(&mtime_part, 4, 1, file_) < 1) return false; for (int i = 0; i < node_count; ++i) { id = nodes[i]->id(); @@ -217,7 +221,9 @@ bool DepsLog::Load(const string& path, State* state, string* err) { assert(size % 4 == 0); int* deps_data = reinterpret_cast(buf); int out_id = deps_data[0]; - TimeStamp mtime = reinterpret_cast(&deps_data[1])[0]; + TimeStamp mtime; + mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) | + (uint64_t)(unsigned int)deps_data[1]); deps_data += 3; int deps_count = (size / 4) - 3; diff --git a/src/deps_log.h b/src/deps_log.h index 793af07910..b1aa361833 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -57,8 +57,9 @@ struct State; /// one's complement of the expected index of the record (to detect /// concurrent writes of multiple ninja processes to the log). /// dependency records are an array of 4-byte integers -/// [output path id, output path mtime (8-byte int), input path id, -/// input path id...] +/// [output path id, +/// output path mtime (lower 4 bytes), output path mtime (upper 8 bytes), +/// input path id, input path id...] /// (The mtime is compared against the on-disk output path mtime /// to verify the stored data is up-to-date.) /// If two records reference the same output the latter one in the file From d2bf82fef0535bc351a69392c0e0c5edc3aaf69f Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 30 Jun 2017 23:55:09 -0400 Subject: [PATCH 6/9] Update checks for new stat fields. This uses the macros as defined by the man page, which, as noted in the comments, are defined correctly on as many libc's that I could check. --- src/disk_interface.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index ffa58db94d..f83151c4ff 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -194,12 +194,14 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { #if defined(__APPLE__) && !defined(_POSIX_C_SOURCE) return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL + st.st_mtimespec.tv_nsec); -#elif defined(_LARGEFILE64_SOURCE) +#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ + defined(__BIONIC__)) + // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html + // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above. + // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar + // For bionic, C and POSIX API is always enabled. return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec; -#elif defined(__CYGWIN__) - return (int64_t)st.st_mtime * 1000000000LL; #else - // see http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec; #endif #endif From f117cc16f6ecf991d16737472f75f9328aa423f0 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 1 Jul 2017 03:49:59 -0400 Subject: [PATCH 7/9] Fix some Windows troubles. Add parentheses so that constant does not overflow; include inttypes.h when using MinGW to get the proper macros. --- src/disk_interface.cc | 2 +- src/win32port.h | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index f83151c4ff..4b4c4c7611 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -65,7 +65,7 @@ TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | ((uint64_t)filetime.dwLowDateTime); // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime - 12622770400LL * 1000000000LL / 100; + return (TimeStamp)mtime - 12622770400LL * (1000000000LL / 100); } TimeStamp StatSingleFile(const string& path, string* err) { diff --git a/src/win32port.h b/src/win32port.h index 546479762b..e542536cc7 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -15,6 +15,13 @@ #ifndef NINJA_WIN32PORT_H_ #define NINJA_WIN32PORT_H_ +#if defined(__MINGW32__) || defined(__MINGW64__) +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include +#endif + typedef signed short int16_t; typedef unsigned short uint16_t; /// A 64-bit integer type From df19c5a597991942bb771df7f6450b6eb15f882a Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 16 Sep 2017 05:02:57 -0400 Subject: [PATCH 8/9] Use strtoll when reading mtime from build log. This prevents overflow on Windows where 'long' is not 64-bit. --- src/build_log.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build_log.cc b/src/build_log.cc index a591050d8f..648617c8e7 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -290,7 +290,7 @@ bool BuildLog::Load(const string& path, string* err) { if (!end) continue; *end = 0; - restat_mtime = atol(start); + restat_mtime = strtoll(start, NULL, 10); start = end + 1; end = (char*)memchr(start, kFieldSeparator, line_end - start); From 0c42653da8ccaeb14b7c28ba9670eeac0460037e Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 16 Sep 2017 05:34:29 -0400 Subject: [PATCH 9/9] Fix minor typo. --- src/deps_log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deps_log.h b/src/deps_log.h index b1aa361833..3812a28a80 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -58,7 +58,7 @@ struct State; /// concurrent writes of multiple ninja processes to the log). /// dependency records are an array of 4-byte integers /// [output path id, -/// output path mtime (lower 4 bytes), output path mtime (upper 8 bytes), +/// output path mtime (lower 4 bytes), output path mtime (upper 4 bytes), /// input path id, input path id...] /// (The mtime is compared against the on-disk output path mtime /// to verify the stored data is up-to-date.)