Skip to content

Commit a87e546

Browse files
luisleo526claude
andcommitted
CR-150 o1w-ledger: paged TradeLedger rows, digest checkpoints, windowed scratch (R07-R09)
R07: 64 fixed resident descriptors per ledger; rows and strings live in the external store, mutable tails included, so a resident row costs a fixed word count instead of a Trade. R08: 64 digest-prefix checkpoints; amendments older than the sealed seed replay from it, so history reads are O(window) and not O(T). R09: phase history and permutation scratch use fixed windows; the existing nine-word sealed fold is unchanged. Reader migrations for the touched accessors; tests test_ledger_history_scale (100k), test_ledger_history_scale_400k and the external-reader coverage added to test_trade_ledger. Draft from lane o1w-ledger (Codex gpt-6.1-sol, effort max). No compile, unit test, corpus or parity run yet; lab proof owed. Scope deviations and the full reader list are in O1W_LEDGER_REPORT.md. Co-Authored-By: Claude Code <[email protected]>
1 parent 583c221 commit a87e546

43 files changed

Lines changed: 1706 additions & 301 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎O1W_LEDGER_REPORT.md‎

Lines changed: 296 additions & 0 deletions
Large diffs are not rendered by default.

‎include/pineforge/detail/scratch_file.hpp‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
#include <system_error>
1515
#include <unistd.h>
1616
#include <utility>
17+
#if defined(__linux__)
18+
#include <sys/vfs.h>
19+
#endif
1720

1821
namespace pineforge::detail {
1922

@@ -39,11 +42,21 @@ class ScratchFile {
3942
clear();
4043
fd_ = std::exchange(other.fd_, -1);
4144
directory_ = std::move(other.directory_);
45+
bounded_cache_ = other.bounded_cache_;
46+
dirty_begin_ = other.dirty_begin_;
47+
dirty_end_ = other.dirty_end_;
48+
other.dirty_begin_ = other.dirty_end_ = 0;
4249
}
4350
return *this;
4451
}
4552
~ScratchFile() { clear(); }
4653

54+
// Opt in before opening. Only the ledger/phase consumers request this;
55+
// existing scratch users keep their established I/O behavior.
56+
void bound_cache() {
57+
if (fd_ >= 0) throw std::logic_error("scratch cache policy changed after open");
58+
bounded_cache_ = true;
59+
}
4760
void set_directory(const std::string& value) {
4861
if (fd_ >= 0 && directory_ != value)
4962
throw std::logic_error("scratch directory changed after spill");
@@ -52,6 +65,7 @@ class ScratchFile {
5265
void clear() noexcept {
5366
if (fd_ >= 0) ::close(fd_);
5467
fd_ = -1;
68+
dirty_begin_ = dirty_end_ = 0;
5569
}
5670
void open(const char* prefix) {
5771
if (fd_ >= 0) return;
@@ -74,6 +88,32 @@ class ScratchFile {
7488
throw ScratchIoError(error, "scratch cloexec");
7589
}
7690
fd_ = opened;
91+
if (bounded_cache_) {
92+
#if defined(__linux__)
93+
struct statfs backing{};
94+
if (::fstatfs(fd_, &backing) != 0) {
95+
const int error = errno;
96+
clear();
97+
throw ScratchIoError(error, "scratch backing");
98+
}
99+
// These are memory filesystems, not an external row source.
100+
if (backing.f_type == 0x01021994 || backing.f_type == 0x858458f6) {
101+
clear();
102+
throw ScratchIoError(ENODEV, "scratch requires disk backing");
103+
}
104+
const int error = ::posix_fadvise(fd_, 0, 0, POSIX_FADV_RANDOM);
105+
if (error != 0) {
106+
clear();
107+
throw ScratchIoError(error, "scratch random access");
108+
}
109+
#elif defined(__APPLE__)
110+
if (::fcntl(fd_, F_NOCACHE, 1) == -1) {
111+
const int error = errno;
112+
clear();
113+
throw ScratchIoError(error, "scratch uncached access");
114+
}
115+
#endif
116+
}
77117
}
78118
static void check_extent(std::uint64_t offset, std::size_t length) {
79119
const auto maximum = static_cast<std::uint64_t>(std::numeric_limits<off_t>::max());
@@ -85,8 +125,11 @@ class ScratchFile {
85125
const auto* bytes = static_cast<const unsigned char*>(data);
86126
std::size_t written = 0;
87127
while (written < length) {
88-
const auto chunk = std::min(length - written,
89-
static_cast<std::size_t>(std::numeric_limits<ssize_t>::max()));
128+
const auto position = offset + written;
129+
const auto chunk = std::min(length - written, bounded_cache_
130+
? kCacheBytes - static_cast<std::size_t>(position % kCacheBytes)
131+
: static_cast<std::size_t>(std::numeric_limits<ssize_t>::max()));
132+
prepare_write(position, chunk);
90133
const ssize_t count = ::pwrite(fd_, bytes + written, chunk,
91134
static_cast<off_t>(offset + written));
92135
if (count < 0 && errno == EINTR) continue;
@@ -99,18 +142,48 @@ class ScratchFile {
99142
auto* bytes = static_cast<unsigned char*>(data);
100143
std::size_t received = 0;
101144
while (received < length) {
102-
const auto chunk = std::min(length - received,
103-
static_cast<std::size_t>(std::numeric_limits<ssize_t>::max()));
145+
const auto chunk = std::min(length - received, bounded_cache_ ? kCacheBytes
146+
: static_cast<std::size_t>(std::numeric_limits<ssize_t>::max()));
104147
const ssize_t count = ::pread(fd_, bytes + received, chunk,
105148
static_cast<off_t>(offset + received));
106149
if (count < 0 && errno == EINTR) continue;
107150
if (count <= 0) throw ScratchIoError(count < 0 ? errno : EIO, "scratch read");
108151
received += static_cast<std::size_t>(count);
152+
discard_clean_cache();
109153
}
110154
}
111155

112156
private:
157+
static constexpr std::size_t kCacheBytes = 64 * 1024;
158+
void discard_clean_cache() const {
159+
#if defined(__linux__)
160+
if (bounded_cache_) {
161+
const int error = ::posix_fadvise(fd_, 0, 0, POSIX_FADV_DONTNEED);
162+
if (error != 0) throw ScratchIoError(error, "scratch discard cache");
163+
}
164+
#endif
165+
}
166+
void prepare_write(std::uint64_t offset, std::size_t count) {
167+
if (!bounded_cache_) return;
168+
const auto first = dirty_begin_ == dirty_end_ ? offset : std::min(offset, dirty_begin_);
169+
const auto end = std::max(offset + count, dirty_end_);
170+
if (end - first > kCacheBytes) {
171+
#if defined(__linux__)
172+
int result;
173+
do { result = ::fdatasync(fd_); } while (result != 0 && errno == EINTR);
174+
if (result != 0) throw ScratchIoError(errno, "scratch flush cache window");
175+
#endif
176+
discard_clean_cache();
177+
dirty_begin_ = offset;
178+
dirty_end_ = offset + count;
179+
} else {
180+
dirty_begin_ = first;
181+
dirty_end_ = end;
182+
}
183+
}
113184
int fd_ = -1;
114185
std::string directory_;
186+
bool bounded_cache_ = false;
187+
std::uint64_t dirty_begin_ = 0, dirty_end_ = 0;
115188
};
116189
} // namespace pineforge::detail
Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
#pragma once
2+
3+
#include <pineforge/detail/scratch_file.hpp>
4+
5+
#include <algorithm>
6+
#include <array>
7+
#include <cstddef>
8+
#include <cstdint>
9+
#include <iterator>
10+
#include <limits>
11+
#include <stdexcept>
12+
#include <type_traits>
13+
14+
namespace pineforge::detail {
15+
16+
// A numeric sequence over ScratchFile, with one fixed 64 KiB write-back
17+
// window. Prefix retirement changes an offset, never moves rows or builds
18+
// an in-memory directory. These are exact bytes, not a hash summary.
19+
template<class T>
20+
class ScratchSequence {
21+
static_assert(std::is_trivially_copyable_v<T>);
22+
static_assert(sizeof(T) <= 64 * 1024);
23+
static constexpr std::size_t kValues = 64 * 1024 / sizeof(T);
24+
public:
25+
ScratchSequence() { file_.bound_cache(); }
26+
class const_iterator {
27+
public:
28+
using iterator_category = std::input_iterator_tag;
29+
using value_type = T;
30+
using difference_type = std::ptrdiff_t;
31+
using reference = T;
32+
using pointer = void;
33+
const_iterator() = default;
34+
T operator*() const { return owner_->at(index_); }
35+
const_iterator& operator++() { ++index_; return *this; }
36+
const_iterator operator++(int) { auto old = *this; ++*this; return old; }
37+
const_iterator operator+(std::size_t n) const { return {owner_, index_ + n}; }
38+
bool operator==(const const_iterator& o) const {
39+
return owner_ == o.owner_ && index_ == o.index_;
40+
}
41+
bool operator!=(const const_iterator& o) const { return !(*this == o); }
42+
private:
43+
friend class ScratchSequence;
44+
const_iterator(const ScratchSequence* owner, std::size_t index)
45+
: owner_(owner), index_(index) {}
46+
const ScratchSequence* owner_ = nullptr;
47+
std::size_t index_ = 0;
48+
};
49+
50+
void set_directory(const std::string& directory) {
51+
file_.set_directory(directory);
52+
directory_ = directory;
53+
}
54+
std::size_t size() const noexcept { return end_ - base_; }
55+
bool empty() const noexcept { return size() == 0; }
56+
static constexpr std::size_t resident_capacity() noexcept { return kValues; }
57+
void clear() noexcept {
58+
file_.clear();
59+
base_ = end_ = page_ = loaded_ = 0;
60+
dirty_ = false;
61+
}
62+
T at(std::size_t index) const {
63+
if (index >= size()) throw std::out_of_range("scratch sequence index");
64+
load(base_ + index);
65+
return window_[(base_ + index) % kValues];
66+
}
67+
T operator[](std::size_t index) const { return at(index); }
68+
void set(std::size_t index, T value) {
69+
if (index >= size()) throw std::out_of_range("scratch sequence index");
70+
load(base_ + index);
71+
window_[(base_ + index) % kValues] = value;
72+
dirty_ = true;
73+
}
74+
void push_back(T value) {
75+
if (end_ == std::numeric_limits<std::size_t>::max())
76+
throw ScratchIoError(EFBIG, "scratch sequence extent");
77+
ScratchFile::check_extent(byte_offset(end_), sizeof(T));
78+
load(end_);
79+
window_[end_ % kValues] = value;
80+
loaded_ = std::max(loaded_, end_ % kValues + 1);
81+
dirty_ = true;
82+
++end_;
83+
}
84+
void resize(std::size_t count, T value = T{}) {
85+
while (size() < count) push_back(value);
86+
if (count == size()) return;
87+
flush();
88+
end_ = base_ + count;
89+
loaded_ = 0;
90+
}
91+
const_iterator begin() const { return {this, 0}; }
92+
const_iterator end() const { return {this, size()}; }
93+
void erase(const_iterator first, const_iterator last) {
94+
if (first.owner_ != this || last.owner_ != this
95+
|| first.index_ != 0 || last.index_ > size())
96+
throw std::logic_error("scratch sequence erase is not a prefix");
97+
base_ += last.index_;
98+
}
99+
template<class Compare>
100+
void stable_sort(Compare before) {
101+
ScratchSequence scratch;
102+
scratch.set_directory(directory_);
103+
for (std::size_t width = 1; width < size();) {
104+
scratch.clear();
105+
PageReader left_page(*this), right_page(*this);
106+
for (std::size_t start = 0; start < size();) {
107+
const auto middle = start + std::min(width, size() - start);
108+
const auto stop = middle + std::min(width, size() - middle);
109+
auto left = start, right = middle;
110+
while (left < middle || right < stop) {
111+
if (right == stop || (left < middle
112+
&& !before(right_page.at(right), left_page.at(left))))
113+
scratch.push_back(left_page.at(left++));
114+
else
115+
scratch.push_back(right_page.at(right++));
116+
}
117+
start = stop;
118+
}
119+
using std::swap;
120+
swap(*this, scratch);
121+
if (width > size() / 2) break;
122+
width *= 2;
123+
}
124+
}
125+
private:
126+
class PageReader {
127+
public:
128+
explicit PageReader(const ScratchSequence& source) : source_(source) {}
129+
T at(std::size_t index) {
130+
if (index < first_ || index - first_ >= count_) {
131+
source_.flush();
132+
const auto first = index - index % kValues;
133+
const auto count = std::min(kValues, source_.size() - first);
134+
count_ = 0;
135+
source_.file_.read(byte_offset(source_.base_ + first),
136+
values_.data(), count * sizeof(T));
137+
first_ = first;
138+
count_ = count;
139+
}
140+
return values_[index - first_];
141+
}
142+
private:
143+
const ScratchSequence& source_;
144+
std::array<T, kValues> values_{};
145+
std::size_t first_ = 0, count_ = 0;
146+
};
147+
static std::uint64_t byte_offset(std::size_t index) {
148+
if (index > std::numeric_limits<std::uint64_t>::max() / sizeof(T))
149+
throw ScratchIoError(EFBIG, "scratch sequence offset");
150+
return static_cast<std::uint64_t>(index) * sizeof(T);
151+
}
152+
void flush() const {
153+
if (!dirty_) return;
154+
file_.open("pineforge-sequence");
155+
file_.write(byte_offset(page_), window_.data(), loaded_ * sizeof(T));
156+
dirty_ = false;
157+
}
158+
void load(std::size_t global) const {
159+
const std::size_t next = global - global % kValues;
160+
if (loaded_ != 0 && page_ == next) return;
161+
flush();
162+
const auto count = next < end_ ? std::min(kValues, end_ - next) : 0;
163+
loaded_ = 0;
164+
if (count != 0)
165+
file_.read(byte_offset(next), window_.data(), count * sizeof(T));
166+
page_ = next;
167+
loaded_ = count;
168+
}
169+
mutable ScratchFile file_;
170+
std::string directory_;
171+
mutable std::array<T, kValues> window_{};
172+
std::size_t base_ = 0, end_ = 0;
173+
mutable std::size_t page_ = 0, loaded_ = 0;
174+
mutable bool dirty_ = false;
175+
};
176+
} // namespace pineforge::detail

‎include/pineforge/engine.hpp‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1813,16 +1813,21 @@ class BacktestEngine {
18131813
MagnifierDistribution magnifier_dist = MagnifierDistribution::ENDPOINTS);
18141814

18151815
int trade_count() const { return (int)trades_.size(); }
1816-
const Trade& get_trade(int i) const { return trades_[i]; }
1816+
Trade get_trade(int i) const { return trades_.read(static_cast<std::size_t>(i)); }
18171817
// The same closed rows under the `closed_trade_*` name a host reads them
18181818
// by (RP6). The per-field `closed_trade_*` family below stays protected
1819-
// Pine plumbing: one whole row by reference answers all of it, and a host
1819+
// Pine plumbing: one owning row value answers all of it, and a host
18201820
// that owns its report needs the row, not twenty wrappers. Range-end
18211821
// report rows are NOT here — report_trade_count() / get_report_trade()
1822-
// below span both spaces. Unchecked, like get_trade above: index against
1823-
// closed_trade_count().
1822+
// below span both spaces. Index against closed_trade_count(); each read
1823+
// owns its exact value, including a row below the published base.
18241824
std::size_t closed_trade_count() const noexcept { return trades_.size(); }
1825-
const Trade& closed_trade(std::size_t i) const { return trades_[i]; }
1825+
Trade closed_trade(std::size_t i) const { return trades_.read(i); }
1826+
// Newest first, preserving global indices across resident and external
1827+
// rows. Start cursor at closed_trade_count(); false means cursor == 0.
1828+
bool previous_closed_trade(std::size_t& cursor, Trade& value) const {
1829+
return trades_.previous(cursor, value);
1830+
}
18261831
// The REPORT's row space: trades_ followed by range_end_trades_, in the
18271832
// order fill_trades_section lays pf_report_t::trades out. trade_count()
18281833
// / get_trade() stay the Pine-visible closed trades (strategy.closedtrades
@@ -1832,11 +1837,19 @@ class BacktestEngine {
18321837
int report_trade_count() const {
18331838
return (int)(trades_.size() + range_end_trades_.size());
18341839
}
1835-
const Trade& get_report_trade(int i) const {
1840+
Trade get_report_trade(int i) const {
18361841
const int n_closed = (int)trades_.size();
18371842
return i < n_closed ? trades_[(size_t)i]
18381843
: range_end_trades_[(size_t)(i - n_closed)];
18391844
}
1845+
// The legacy C string ABI promises pointer lifetime until the next run.
1846+
// Asking for that borrowed storage explicitly materializes the requested
1847+
// row. Ordinary report/value readers never enter this compatibility path.
1848+
const Trade& materialize_report_trade(int i) const {
1849+
const int n_closed = static_cast<int>(trades_.size());
1850+
return i < n_closed ? trades_.materialize(static_cast<std::size_t>(i))
1851+
: range_end_trades_[static_cast<std::size_t>(i - n_closed)];
1852+
}
18401853

18411854
// ABI v4 live-runtime surface (task 9): classify why a REPORT-row
18421855
// closed trade exited -- report-row scope (spans trades_ then

‎include/pineforge/pineforge.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1223,7 +1223,8 @@ PF_API int64_t strategy_position_cycle_seq(pf_strategy_t s);
12231223
* (`trades_` then the range-end rows, `open_at_end`). The returned pointer
12241224
* is valid until the next run() (or stream call) on this handle, like
12251225
* #strategy_get_last_error. NULL on a NULL @p s or an out-of-range
1226-
* @p trade_index. */
1226+
* @p trade_index. This legacy borrowed-pointer API materializes each
1227+
* requested row until the next run; full-history requests use growing RAM. */
12271228
PF_API const char* strategy_closed_trade_entry_id(pf_strategy_t s, int trade_index);
12281229
/** See #strategy_closed_trade_entry_id for the row-space/lifetime/NULL
12291230
* contract. The returned string is the engine's own INTERNAL exit id, not

‎include/pineforge/report_sink.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ typedef struct pf_report_sink_v1 {
3535
* The seven counts correspond to E, T, O, V, K, X, H in that order.
3636
* sha256_hex is the trailer's lowercase SHA-256, NUL-terminated, and is ""
3737
* until the trailer exists. complete becomes 1 once the trailer is written.
38-
* trades_retained is 1 when closedtrades.* history also stays in RAM.
38+
* trades_retained is 1 when closedtrades.* history stays script-readable.
3939
* The explicit seven-byte reserved field gives a 144-byte layout with no
4040
* implicit padding. PF_ABI_VERSION stays 4; no existing report/trade grows.
4141
* RESOLVED S1: receipt query initialization and outcomes follow the

0 commit comments

Comments
 (0)