### ----------------------------------------------------
### -- PDF
### -- Licenses: Artistic-2.0
### -- Authors: David Warring
### -- File: lib/PDF/IO/Reader.rakumod
### ----------------------------------------------------
### -- Chunk 1 of 2
unit class PDF::IO::Reader;

use PDF::COS;
use PDF::Grammar:ver<0.2.1+>;
use PDF::Grammar::COS;
use PDF::Grammar::PDF;
use PDF::Grammar::PDF::Actions;
use PDF::IO;
use PDF::IO::Crypt::PDF;
use PDF::IO::IndObj;
use PDF::IO::Serializer;
use PDF::COS :IndRef, :IndObj;
use PDF::COS::Dict;
use PDF::COS::Util :from-ast, :to-ast;
use PDF::IO::Writer;
use X::PDF;
use Hash::int;
use JSON::Fast;
use Method::Also;

constant GenNumMax = 100_000;

subset ObjNumInt of UInt;
subset GenNumInt of Int where 0 <= * < GenNumMax;
subset StreamAstNode of Pair:D where .key eq 'stream';

has PDF::IO  $.input is rw;      #= raw PDF image (latin-1 encoding)
has Str      $.file-name;
has          %!ind-obj-idx is Hash::int; # keys are: $obj-num*GenNumMax + $gen-num
has Bool     $.auto-deref is rw = True;
has Rat      $.version is rw;
has Str      $.type is rw;       #= 'PDF', 'FDF', etc...
has uint64   $.prev;             #= xref offset
has uint     $.size is rw;       #= /Size entry in trailer dict ~ first free object number
has uint64   @.xrefs = (0);      #= xref position for each revision in the file
has PDF::IO::Crypt::PDF $.crypt is rw;
has Rat $.compat;        #= cross reference stream mode
method compat is rw {
    Proxy.new:
        FETCH => { $!compat // $!version // 1.4 },
        STORE => -> $, $!compat {}
    ;
}
has Lock $!lock .= new;

my enum IndexType <Free External Embedded>;

# faster native method overrides
role NativeReader[$native-reader] {
    #| load PDF 1.4- xref table followed by trailer
    method load-xref-table(Str $xref is copy, $dict is rw, :$offset) {
        # fast load of the xref segments
        my $buf = $xref.encode("latin-1");
        my array $entries = $native-reader.read-xref($buf)
            // die X::PDF::BadXRef::Parse.new( :$offset, :$xref );
        my $bytes = $native-reader.xref-bytes;

        # parse and load the trailer
        my $trailer = $buf.subbuf($bytes).decode("latin-1");
        my $parse = PDF::Grammar::COS.subparse( $trailer.trim, :rule<trailer>, actions => $.scan-actions );
        die X::PDF::BadXRef::Parse.new( :$offset, :$xref )
            unless $parse;
        my \index = $parse.ast;
        $dict = PDF::COS.coerce( |index<trailer>, :reader(self) );

        my uint64 @seg[+$entries div 4;4] Z= @$entries;

        [@seg, ];
    }
}

# further COS object native method overrides
role NativeCOS[$cos-node, $cos-ind-obj] {
    method parse-ind-obj(Str:D $input) {
        $cos-ind-obj.parse($input)
    }
    method parse-object(Str:D $input) {
        $cos-node.parse($input)
    }
    my class COSScanActions {
        has Bool $.get-offsets is rw = False; #| return ind-obj byte offsets in AST
        method TOP($/) { make $<cos>.ast }
        method cos($/) {
            my @body = @<body>».ast;
            make (:@body);
        }
        method body($/) {
            my @objects = @<ind-obj>».ast;
            my %body = :@objects;
            %body ,= .<trailer>.ast with $<index>;
            make %body;
        }
        method ind-obj($/) {
            my Str:D() $input = $/;
            my $ind-obj = $cos-ind-obj.parse($input, :scan)
                // die "Native indirect Object parse failed at byte offset:{$/.from}";
            my $ast = $ind-obj.ast;
            $ast.value.push: $/.from
                if self.get-offsets;
            make $ast;
        }
        method trailer($/) {
            my $dict = $cos-node.parse: $<dict>.Str
               // die "Native trailer dictionary parse failed at byte offset:{$/.from}";
            make (:trailer($dict.ast))
        }
    }
    method scan-actions { COSScanActions.new }
}

submethod TWEAK(PDF::COS::Dict :$trailer) {
    self!install-trailer($_) with $trailer;

    try {
        require ::('PDF::Native::Reader');
        self does NativeReader[::('PDF::Native::Reader').new];
    }
    try {
        require ::('PDF::Native::COS');
        self does NativeCOS[::('PDF::Native::COS::COSNode'), ::('PDF::Native::COS::COSIndObj'), ];
    }
}

method actions is also<scan-actions> {
    PDF::Grammar::PDF::Actions.new: :lite;
}

method trailer is rw {
    sub FETCH($) {
        # vivify
        self!install-trailer(PDF::COS::Dict.new: :reader(self))
            unless %!ind-obj-idx{0}:exists;
        self.ind-obj(0, 0).object;
    }
    sub STORE($, \obj) {
        self!install-trailer(obj);
    }
    Proxy.new: :&FETCH, :&STORE;
}

method !install-trailer(PDF::COS::Dict $object) {
    %!ind-obj-idx{0} = do {
        my PDF::IO::IndObj $ind-obj .= new( :$object, :obj-num(0), :gen-num(0) );
        %( :type(IndexType::External), :$ind-obj );
    }
}

method !setup-crypt(Str :$password = '') {
    my Hash $doc = self.trailer;
    with $doc<Encrypt> -> \enc {
        $!crypt = PDF::COS.required('PDF::IO::Crypt::PDF').new( :$doc );
        $!crypt.authenticate( $password );
        my \enc-obj-num = enc.obj-num // -1;
        my \enc-gen-num = enc.gen-num // -1;

        for %!ind-obj-idx.kv -> $k, Hash:D $idx {
            my ObjNumInt $obj-num = $k div GenNumMax
                or next;
            my GenNumInt $gen-num = $k mod GenNumMax;

            # skip the encryption dictionary, if it's an indirect object
            unless $obj-num == enc-obj-num
                && $gen-num == enc-gen-num {
                # decrypt all objects that have already been loaded
                without $idx<encrypted> {
                    $_ = True;
                    with $idx<ind-obj> -> $ind-obj {
                        die "too late to setup encryption: $obj-num $gen-num R"
                        if $idx<type> != Free | External
                        || $ind-obj.isa(PDF::IO::IndObj);

                        $!crypt.crypt-ast( (:$ind-obj), :$obj-num, :$gen-num, :mode<decrypt> );
                    }
                }
            }
        }
    }
}

#| [PDF 32000 Table 15] Entries in the file trailer dictionary
method !set-trailer (
    Hash $dict,
    Array :$keys = [ $dict.keys.grep: {
        $_ !~~ 'Prev'|'Size'                    # Recomputed fields
           |'Type'|'DecodeParms'|'Filter'|'Index'|'W'|'Length'|'XRefStm' # Unwanted, From XRef Streams
    } ],
    ) {
    temp $.auto-deref = False;
    my Hash $trailer = self.trailer;

    for $keys.sort -> \k {
        $trailer{k} = from-ast $_
             with $dict{k};
    }

    $trailer;
}

#| open the named PDF/FDF file
multi method open( Str $!file-name where {!.does(PDF::IO)}, |c) is hidden-from-backtrace {
    $.open( $!file-name.IO, |c );
}

#| deserialize a JSON dump
multi method open(IO::Path $input-path  where .extension.lc eq 'json', |c ) is hidden-from-backtrace {
    my Str $input-file = $input-path.absolute;
    my \root = $input-path.IO.slurp.&from-json;
    my \ast = root<cos> if root.isa(Hash);
    die X::PDF::BadJSON.new( :$input-file )
        without ast;
    self.open: ast, |c;
}

multi method open(Associative \ast is raw, Str :$input-file, |c) is hidden-from-backtrace {
    $!type = ast<header><type> // 'PDF';
    $!version = ast<header><version> // 1.2;
    for ast<body>.list {

        for .<objects>.list.reverse {
            with .<ind-obj> -> $ind-obj {
                (my ObjNumInt $obj-num, my GenNumInt $gen-num) = $ind-obj.list;
                my $k := $obj-num * GenNumMax + $gen-num;
                %!ind-obj-idx{$k} = %(
                    :type(IndexType::External),
                    :$ind-obj,
                ) unless %!ind-obj-idx{$k}:exists;
            }
        }

        with .<trailer> {
            my Hash $dict = PDF::COS.coerce( |$_ );
            self!set-trailer( $dict.content<dict> );
            self!setup-crypt(|c);
        }
   }

    ast;
}

# process a batch of indirect object updates
method update-index( :@entries!, UInt :$!prev, UInt :$!size ) {
    @!xrefs.push: $!prev;

    for @entries -> Hash $entry {
        my ObjNumInt $obj-num = $entry<obj-num>
            or next;

        my GenNumInt $gen-num = $entry<gen-num>;
        my UInt $type = $entry<type>;
        my $k := $obj-num * GenNumMax + $gen-num;

        given $type {
            when IndexType::Free {
                %!ind-obj-idx{$k}:delete;
            }
            when IndexType::External {
                my $ind-obj = $entry<ind-obj>;
                %!ind-obj-idx{$k} = %(
                    :$type,
                    :$ind-obj,
                );
            }
            default {
                die "unable to handle indirect object update of type: $_";
            }
        }
    }
}

multi method open(PDF::IO() $!input!, |c) is hidden-from-backtrace {
    $.load-header( );
    $.load-cos( $.type, |c );
}

#| load the data for a stream object. Cross check actual size versus expected /Length
method !fetch-stream-data(@ind-obj,           #| primary object
                          $input,             #| associated input stream
                          UInt:D :$offset,    #| offset of the object in the input stream
                          UInt:D :$obj-len,   #| upper bound for the end of the stream
    )
{
    my constant MinTrailingBytes = "\endstream endobj".codes;
    my (ObjNumInt $obj-num, GenNumInt $gen-num, $obj-raw) = @ind-obj;
    my UInt() $from = $obj-raw.value<start>:delete;
    my UInt() $length = $.deref( $obj-raw.value<dict><Length> )
        // die X::PDF::BadIndirectObject.new(
               :$obj-num, :$gen-num, :$offset,
               :details("Stream mandatory /Length field is missing")
    );

    my $max-remaining-bytes = $obj-len - $from;
    if $length > $max-remaining-bytes - MinTrailingBytes {
        my $details := $max-remaining-bytes < MinTrailingBytes
            ?? "Stream data is truncated"
            !! "Stream dictionary entry /Length $length is too long for containing indirect object (maximum size here is {$max-remaining-bytes - MinTrailingBytes} bytes)";
        die X::PDF::BadIndirectObject.new: :$obj-num, :$gen-num, :$offset, :$details;
    }

    my $stream := $obj-raw.value<encoded> //= do {
        # ensure stream is followed by an 'endstream' marker
        my Str \tail = $input.byte-str( $offset + $from + $length, $max-remaining-bytes - $length);

        if tail ~~ m{^<PDF::Grammar::COS::stream-tail> 'endobj'} {
            warn X::PDF::BadIndirectObject.new(
                :$obj-num, :$gen-num, :$offset,
                :details("Ignoring {$/.from} bytes before 'endstream' marker")
                ) if $/.from;
        }
        else {
            die X::PDF::BadIndirectObject.new(
                :$obj-num, :$gen-num, :$offset,
                :details("Unable to locate 'endstream' 'endobj' sequence after consuming /Length $length bytes")
                );
        }

        $length
            ?? $input.byte-str( $offset + $from, $length )
            !! '';
    };
}

method parse-ind-obj(Str:D $input) {
    PDF::Grammar::COS.subparse( $input, :$.actions, :rule<ind-obj-nibble> );
    $/;
}

method parse-object(Str:D $input) {
    PDF::Grammar::COS.subparse( trim($input), :$.actions, :rule<object> );
    $/;
}

#| type-1: fetch as a top level object from the pdf
 multi method fetch-ind-obj(
     :$type! where IndexType::External, :$offset!, :$end!,   # type-1
     :$obj-num, :$gen-num, :$encrypted = True) {

     die X::PDF::BadXRef::Entry.new: :details("Invalid cross-reference offset $offset for $obj-num $gen-num R")
        unless $offset > 0;

     my UInt $obj-len = do given $end - $offset {
         when 0     { X::PDF::BadXRef::Entry.new: :details("Duplicate cross-reference destination (byte offset $offset) for $obj-num $gen-num R")}
         when * < 0 { die X::PDF::BadXRef::Entry.new: :details("Attempt to fetch object $obj-num $gen-num R at byte offset $offset, past end of PDF ($end bytes)") }
         default    { $_ }
     }

     my Str $input = $!input.byte-str( $offset, $obj-len );
     my @ind-obj = .ast.value given self.parse-ind-obj($input)
         or die X::PDF::BadIndirectObject::Parse.new( :$obj-num, :$gen-num, :$offset, :$input);

     die X::PDF::BadXRef::Entry::Number.new( :$obj-num, :$gen-num, :@ind-obj )
        unless $obj-num == @ind-obj[0] && $gen-num == @ind-obj[1];

     self!fetch-stream-data(@ind-obj, $!input, :$offset, :$obj-len)
         if @ind-obj[2] ~~ StreamAstNode;

     if $encrypted {
         .crypt-ast( (:@ind-obj), :$obj-num, :$gen-num, :mode<decrypt> )
             with $!crypt;
     }

     return @ind-obj;
 }

#| type-2: dereference and extract from the containing object
multi method fetch-ind-obj(
    :$type! where IndexType::Embedded,
    :$index!, :$ref-obj-num!,  # type-2
    :$obj-num, :$gen-num) {

    my subset ObjStm of Hash where { .<Type> ~~ 'ObjStm' }

    my ObjStm \container-obj = $.ind-obj( $ref-obj-num, 0 ).object;
    my \embedded-objects = container-obj.decoded;
    my :($actual-obj-num, $input) := embedded-objects[$index];
    my @ind-obj := [ $actual-obj-num, 0, Any ];
    die X::PDF::BadXRef::Entry::Number.new( :$obj-num, :$gen-num, :@ind-obj )
        unless $obj-num == @ind-obj[0] && $gen-num == @ind-obj[1];

    @ind-obj[2] = .ast given self.parse-object($input)
        or die X::PDF::ObjStmObject::Parse.new( :$obj-num, :$input, :$ref-obj-num);

    @ind-obj;
}

#| fetch and stantiate indirect objects. cache against the index
method ind-obj( ObjNumInt $obj-num!, GenNumInt $gen-num!,
                Bool :$get-ast = False,  #| get ast data, not formulated objects
                Bool :$eager = True,     #| fetch object, if not already loaded
    ) {

    my Hash $idx := %!ind-obj-idx{$obj-num * GenNumMax + $gen-num}
        // die "unable to find object: $obj-num $gen-num R";

    $!lock.protect: {
        my $ind-obj;
        my Bool $have-ast = True;
        with $idx<ind-obj> {
            $ind-obj := $_;
            $have-ast := False
                if $ind-obj.isa(PDF::IO::IndObj);
        }
        else {
            return unless $eager;
            $idx<ind-obj> = $ind-obj := self.fetch-ind-obj(|$idx, :$obj-num, :$gen-num);
        }

        if $get-ast {
            # AST requested.
            $have-ast ?? :$ind-obj !! $ind-obj.ast;
        }
        else {
            # Object requested.
            $have-ast
                ?? ($idx<ind-obj> = PDF::IO::IndObj.new( :$ind-obj, :reader(self) ))
                !! $ind-obj;
        }
    }
}

#| raw fetch of an object, without indexing or decryption
method get(ObjNumInt $obj-num, GenNumInt $gen-num) {
    my %idx := %!ind-obj-idx{$obj-num * GenNumMax + $gen-num}
        // die "unable to find object: $obj-num $gen-num R";
     self.fetch-ind-obj(|%idx, :!encrypted, :$obj-num, :$gen-num);
}

#| utility method for basic deferencing, e.g.
#| $reader.deref($root,<Pages>,<Kids>,[0],<Contents>)
method deref($val is copy, **@ops ) is rw {
    for @ops -> \op {
        $val = self!ind-deref($val)
            if $val.isa(Pair);
        $val = do given op {
            when Str   { $val{ $_ } }
            when UInt  { $val[ $_ ] }
            when Array { $val[ .[0] ] }
            default    {die "bad \$.deref arg: {.raku}"}
        };
    }
    $val = self!ind-deref($val)
        if $val.isa(Pair);
    $val;
}

method !ind-deref(Pair $_! ) {
    return .value unless $_ ~~ IndRef;
    my ObjNumInt \obj-num = .value[0];
    my GenNumInt \gen-num = .value[1];
    $.ind-obj( obj-num, gen-num ).object;
}

method load-header() {
    # file should start with: %PDF-n.m, (where n, m are single
    # digits giving the major and minor version numbers).

    my Str $preamble = $!input.byte-str(0, 32);
    my $actions = $.actions;

    PDF::Grammar::COS.subparse($preamble, :$actions, :rule<header>)
        or PDF::Grammar::COS.subparse($preamble ~ $!input.byte-str(32, 1024), :$actions, :rule<header>)
        or die X::PDF::BadHeader.new( :$preamble );
    given $/.ast {
        $!version = .<version>;
        $!type = .<type>;
    }
}

#| Scan the PDF, bypass any indices or stream lengths
multi method load-cos('PDF', :$repair! where .so, |c ) {
    self!full-scan( PDF::Grammar::PDF, $.scan-actions, :repair, |c )
}

#| Regular PDF load via indirect object indices
multi method load-cos('PDF', |c ) {
    self!load-xrefs( PDF::Grammar::PDF, $.actions, |c );
}

#| Load input in FDF (Form Data Definition) format.
#| Use full-scan mode, as these are not indexed.
multi method load-cos('FDF') {
    self!full-scan(PDF::COS.required('PDF::Grammar::FDF'), $.actions);
}

#| Load of miscellaneous file in COS format
multi method load-cos($type, |c) {
    self!load-xrefs(PDF::Grammar::COS, $.actions, |c );
}

method !locate-xref($offset is copy) {
    my str $xref;
    constant SIZE = 4096;       # big enough to usually contain xref
    my UInt:D $input-bytes := $!input.codes;
    # scan for '%%EOF' marker at the end of the trailer
    $xref = '';
    my $n = 0;
    repeat {
        my UInt $len = min(SIZE * ++$n, $input-bytes - $offset);
        $xref ~= $!input.byte-str( $offset, $len );
        $offset += $len;
    } until $xref.contains('%%EOF') || $offset >= $input-bytes;

    $xref;
}

#| load PDF 1.4- xref table followed by trailer
method load-xref-table(Str $xref, $dict is rw, :$offset) {
    my $parse = PDF::Grammar::COS.subparse( $xref, :rule<index>, :$.actions );
    die X::PDF::BadXRef::Parse.new( :$offset, :$xref )
        unless $parse;

    my \index = $parse.ast;

    $dict = PDF::COS.coerce( |index<trailer>, :reader(self) );

    index<xref>».<entries>;
}

#| load a PDF 1.5+ XRef Stream
method !load-xref-stream(Str $xref, $dict is rw, UInt :$offset, :@discarded) {
    my $parse = PDF::Grammar::COS.subparse($xref, :$.actions, :rule<ind-obj>);

    die X::PDF::BadIndirectObject::Parse.new( :$offset, :input($xref))
        unless $parse;

    my %ast = $parse.ast;
    my PDF::IO::IndObj $ind-obj .= new( |%ast, :input($xref), :reader(self) );
    my subset XRefLike of Hash where { .<Type> ~~ 'XRef' }
    $dict = my XRefLike $ = $ind-obj.object;
    # we don't want to index these
    @discarded.push: $ind-obj.obj-num * GenNumMax + $ind-obj.gen-num;
    $dict.decode-index;
}

method !load-xref-section(UInt:D $offset, :@ends!) {
    my UInt \tail-bytes = min(1024, $!input.codes);
    my array @obj-idx; # array of shaped arrays
    my UInt @discarded;
    my Hash $dict;
    my Str \xref = self!locate-xref($offset);
    if xref ~~ m:s/^ xref/ {
        # traditional 1.4 cross reference index
        @obj-idx.append: self.load-xref-table( xref, $dict, :$offset);
        with $dict<XRefStm> {
            # hybrid 1.4 / 1.5 with a cross-reference stream
            # that contains additional objects
            my $xref-dict = {};
            my Str \xref-stm = self!locate-xref($_);
            @obj-idx.push: self!load-xref-stream(xref-stm, $xref-dict, :offset($_), :@discarded);
        }
        $!compat = 1.4 if $!compat > 1.4;
    }
    else {
        # PDF 1.5+ cross reference stream.
        # need to write index in same format for Adobe reader (issue #22)
        @obj-idx.push: self!load-xref-stream(xref, $dict, :$offset, :@discarded);
        $!compat = 1.5 if $!compat < 1.5;
    }

    enum ( :ObjNum(0), :Type(1),
           :Offset(2), :GenNum(3),     # Type 1 (External) Objects
           :RefObjNum(2), :Index(3)    # Type 2 (Embedded) Objects
         );

    for @obj-idx {
        for ^.elems -> $i {
            my $type := .[$i;Type];

            if $type == IndexType::Embedded {
                my UInt      $index       := .[$i;Index];
                my ObjNumInt $ref-obj-num := .[$i;RefObjNum];
                my $k := .[$i;ObjNum] * GenNumMax;
                %!ind-obj-idx{$k} = %( :$type, :$index, :$ref-obj-num )
                    unless %!ind-obj-idx{$k}:exists;
            }
            elsif $type == IndexType::External {
                my $k := .[$i;ObjNum] * GenNumMax + .[$i;GenNum];
                my $offset = .[$i;Offset];
                %!ind-obj-idx{$k} = %( :$type, :$offset )
                    unless %!ind-obj-idx{$k}:exists;
                @ends.push: $offset;
            }
        }
    }

    %!ind-obj-idx{$_}:delete for @discarded;
    $!size  = do with $dict<Size> { $_ } else { 1 };
    self!set-trailer: $dict;
    $dict<Prev> // Int;
}

#| scan indices, starting at PDF tail. objects can be loaded on demand,
#| via the $.ind-obj() method.
method !load-xrefs($grammar, $actions, |c) {
    my UInt \tail-bytes = min(1024, $!input.codes);
    my Str $tail = $!input.byte-str(* - tail-bytes);
    @!xrefs = [];

    $grammar.parse($tail, :$actions, :rule<postamble>)
        or do {
            CATCH { default {die X::PDF::BadTrailer.new( :$tail ); } }
            # unable to find 'startxref'
            # see if the PDF can be loaded sequentially
            return self!full-scan( $grammar, $actions, |c )
        }

    $!prev = $/.ast<startxref>;
    $!compat = $!version // 1.4;

    my UInt @ends;
    my UInt $offset = $!prev;
    my UInt %visited;

    while $offset.defined {
        die "xref '/Prev' cycle detected in cross-reference tables: \@$offset"
            if %visited{$offset}++;
        @!xrefs.unshift: $offset;
        $offset = self!load-xref-section: $offset, :@ends;
    }

    #| don't entirely trust /Size entry in trailer dictionary
    my ObjNumInt \actual-size = max( %!ind-obj-idx.keys ) div GenNumMax;
    $!size = actual-size + 1
        if $!size <= actual-size;

    # constrain indirect objects to a maximum end position
    @ends.append: @!xrefs;
    @ends.push: $!input.codes;
    @ends .= sort;

    # mark end positions of external objects
    my int $i = 0;
    my int $n = +@ends;
    for %!ind-obj-idx.pairs.grep(*.value<offset>).sort(*.value<offset>) {
        my $v := .value;
        repeat {
            $v<end> = @ends[$i];
        } until $v<end> > $v<offset> || ++$i >= $n;
        # cull, if freed
        %!ind-obj-idx{.key}:delete
            unless $v<type>;
    }

    self!setup-crypt(|c);
}

#| differentiate update xrefs from hybrid xrefs
method revision-xrefs {
    my UInt @updates;
    for @!xrefs {
        @updates.push: $_
            if !@updates || $_ > @updates.tail;
    }
    @updates;
}

#| bypass any indices. directly parse and reconstruct index from objects.
method !full-scan( $grammar, $actions, Bool :$repair, |c) {
    temp $actions.get-offsets = True;
    my Str $input = ~$!input;
    $grammar.parse($input, :$actions)
        or die X::PDF::ParseError.new( :$input );

    my %ast = $/.ast;
    my Hash @body = %ast<body>.list;

    for @body.reverse {
        my Pair @objects = .<objects>.list;

        for @objects.reverse {
            next unless .key eq 'ind-obj';
            my @ind-obj = .value.list;
            my (ObjNumInt $obj-num, GenNumInt $gen-num, $object, UInt $offset) = @ind-obj;

            my $stream-type;

            if $object ~~ StreamAstNode {
                my \stream = $object.value;
                my Hash \dict = stream<dict>;
                $stream-type = .value with dict<Type>;

                # reset/repair stream length
                dict<Length> = :int(stream<encoded>.codes)
                    if $repair;

                if $stream-type ~~ 'XRef' {
                    self!set-trailer( dict, :keys[<Root Encrypt Info ID>] );
                    self!setup-crypt(|c);
                    # discard existing /Type /XRef stream objects. These are specific to the input PDF
                    next;
                }
            }

            my $k := $obj-num * GenNumMax + $gen-num;
            %!ind-obj-idx{$k} = %(
                :type(IndexType::External),
                :@ind-obj,
                :$offset,
            ) unless %!ind-obj-idx{$k}:exists;

            with $stream-type {
                when 'ObjStm' {
                    # Object Stream. Index contents as type 2 objects
                    my \container-obj = $.ind-obj( $obj-num, $gen-num ).object;
                    my Array \embedded-objects = container-obj.decoded;
                    for embedded-objects.kv -> $index, $_ {
                        my ObjNumInt $sub-obj-num = .[0];
                        my ObjNumInt $ref-obj-num = $obj-num;
                        my $k := $sub-obj-num * GenNumMax;
                        %!ind-obj-idx{$k} = %(
                            :type(IndexType::Embedded),
                            :$index,
                            :$ref-obj-num,
                        ) unless %!ind-obj-idx{$k}:exists;
                    }
                }
            }
        }

        with .<trailer> {
            my Hash \trailer = PDF::COS.coerce( |$_ );
            self!set-trailer( trailer.content<dict> );
            self!setup-crypt(|c);
        }
        else {
            die X::PDF::NoTrailer.new
                unless self.trailer;
        }
    }

    %ast;
}

#| Get a list of indirect objects in the PDF
#| - preserve input order
#| - sift /XRef and /ObjStm objects,
method get-objects(
    Bool :$incremental = False,     #| only return updated objects
    Bool :$eager = ! $incremental,  #| fetch uncached objects
    ) {
    my @object-refs;
    for %!ind-obj-idx.keys.sort {
        my Hash:D $entry = %!ind-obj-idx{$_};
        my ObjNumInt $obj-num = $_ div GenNumMax;
        my GenNumInt $gen-num = $_ mod GenNumMax;

        my UInt $seq = 0;
        my UInt $offset;

        given $entry<type> {
            when IndexType::External {
                $offset = $_
                    with $entry<offset>
            }
            when IndexType::Embedded {
                my UInt $parent = $entry<ref-obj-num>;
                with %!ind-obj-idx{$parent * GenNumMax} {
                    $offset = .<offset>;
                }
                else {
                    die "unable to find object: $parent 0 R"
                }
                $seq = $entry<index>;
            }
            when IndexType::Free {
                next;
            }
            default {
                die "unknown ind-obj index <type> $obj-num $gen-num: {.raku}"
            }
        }

        my \ast = $.ind-obj($obj-num, $gen-num, :get-ast, :$eager);

        with ast -> IndObj $_ {
            my \obj = .value[2];

            # discard existing /Type /XRef and ObjStm objects.
            with obj<stream> {
                with .<dict><Type> -> \obj-type {
                    next if obj-type<name> ~~ 'XRef'|'ObjStm';
                }
            }
        }
        elsif $incremental {
            # object hasn't been fetched, so cannot have been updated
            next;
        }

        if $incremental && $offset && $obj-num {
            # check updated vs original PDF value.
            my \original-ast = self.fetch-ind-obj(|%!ind-obj-idx{$obj-num * GenNumMax + $gen-num}, :$obj-num, :$gen-num);
            # discard, if not updated
            next if original-ast eqv ast.value;
        }

        $offset //= 0;
        @object-refs.push( ($offset + $seq) => (ast // :copy[$obj-num, $gen-num, self]) );
    }

    # preserve file order
    my @ = @object-refs.list.sort(*.key)».value;
}

#| get just updated objects. return as indirect objects
method get-updates() {
    my List \raw-objects = $.get-objects( :incremental );
    raw-objects.map: {
        my ObjNumInt $obj-num = .value[0];
        my GenNumInt $gen-num = .value[1];
        $.ind-obj($obj-num, $gen-num).object;
    };
}

method recompress(Bool:D :$compress = True) {
    # locate and or compress/uncompress stream objects
    # replace deprecated LZW compression with Flate

    for self.get-objects
        .grep(*.key eq 'ind-obj')
        .map(*.value)
        .grep: {.[2] ~~ StreamAstNode} {

        my ObjNumInt \obj-num = .[0];
        my GenNumInt \gen-num = .[1];
        my \stream-dict = .[2].value<dict>;
        my Bool \is-compressed = stream-dict<Filter>:exists;

        next if $compress == is-compressed
        # always recompress LZW (which is deprecated)
        && !($compress && stream-dict<Filter><name> ~~ 'LZWDecode');

        # fully stantiate object and adjust compression
        my PDF::COS \object = self.ind-obj( obj-num, gen-num).object;
        $compress ?? .compress !! .uncompress with object;
    }
}

method ast(::CLASS:D $reader: Bool :$rebuild, |c ) {
