### ----------------------------------------------------
### -- Text::CSV
### -- Licenses: Artistic-2.0
### -- Authors: H.Merijn Brand (Tux)
### -- File: Text-CSV.pod
### ----------------------------------------------------
### -- Chunk 3 of 3
',
       price   => 5,
       product => 'mouse'
       }
   }

If C<out> points to an existing Hash, the data will be added to that instead
(existing records will be overwritten on reading duplicate keys). It returns
the value passed to C<out>:

 my $ref = csv(in => "test.csv", out => %hash, key =>        "code");
 my $ref = csv(in => "test.csv", out => %hash, key => [ ":", "code", "color" ]);

=head3 kh
X<kh>

When using hashes, keep the column names into the arrayref passed, so all
headers are available after the call in the original order.

 my $aoh = csv (in => "file.csv", kh => \my @hdr);

This attribute should be aliassed C<keep-header>.  C<keep_header>,
C<keep-headers>, C<keep_headers> C<keep_column_names>, or
C<keep-column-names> but I couldn't get that to work.

This attribute implies a default of C<auto> for the C<headers> attribute.

=head3 fragment
X<fragment>

Only output the fragment as defined in the L</fragment> method. This option
is ignored when I<generating> C<CSV>. See L</out>.

Combining all of them could give something like

 use Text::CSV qw( csv );
 my @aoh = csv(
     in       => "test.txt",
     encoding => "utf-8",
     headers  => "auto",
     sep_char => "|",
     fragment => "row=3;6-9;15-*",
     );
 say @aoh[15]{Foo};

=head3 sep-set
X<sep-set>
X<sep_set>
X<seps>

If C<sep-set> is set, the method L</header> is invoked on the opened stream
to detect and set L<C<sep>|/sep> with the given set.

C<sep-set> can be abbreviated to C<seps>.

Note that as the method L</header> is invoked, its default is to also set
the headers.

=head3 set_column_names
X<set_column_names>

If  C<set_column_names> is passed,  the method L</header> is invoked on the
opened stream with all arguments meant for L</header>.

=head2 Callbacks

Callbacks enable actions triggered from the I<inside> of Text::CSV.

While most of what this enables can easily be done in an unrolled loop as
described in the L</SYNOPSIS> callbacks, can be used to meet special
demands or enhance the L</csv> function.

All callbacks except C<error> are called with just one argument: the
current L<CSV::Row>.

=over 2

=item error
X<error>

 $csv.callbacks(error => { $csv.SetDiag(0) });

the C<error> callback is invoked when an error occurs, but I<only> when
L</auto_diag> is set to a true value. This callback is invoked with the
values returned by L</error_diag>:

 my ($c, $s);

 sub ignore3006 (Int $err, Str $msg, Int $pos, Int $recno, Int $fldno) {
     if ($err == 3006) {
         # ignore this error
         ($c, $s) = (Str, Str);
         Text::CSV.SetDiag(0);
         }
     # Any other error
     return;
     } # ignore3006

 $csv.callbacks(error => \&ignore3006);
 $csv.bind_columns(\$c, \$s);
 while ($csv.getline($fh)) {
     # Error 3006 will not stop the loop
     }

=item after_parse
X<after_parse>
X<after-parse>

 sub add-new (CSV::Row $r) { $r.fields.push: "NEW"; }
 $csv.callbacks(after_parse => &add-new);
 while (my @row = $csv.getline($fh)) {
     @row[-1] eq "NEW";
     }

This callback is invoked after parsing with L</getline> only if no error
occurred.

The return code of the callback is ignored.

 sub add_from_db (CSV::Row $r) {
     $sth.execute($r[4]);
     push $r.fields: $sth.fetchrow_array;
     } # add_from_db

 my $aoa = csv(in => "file.csv", callbacks => {
     after_parse => &add_from_db });

 my $aoa = csv(in => "file.csv", after_parse => {
     $sth.execute($^row[4]); $^row.fields.push: $sth.fetchrow_array; });

=item before_print
X<before_print>
X<before-print>

 my $idx = 1;
 $csv.callbacks(before_print => { $^row[0] = $idx++ });
 $csv.print($*OUT, [ 0, $_ ]) for @members;

This callback is invoked before printing with L</print> only if no error
occurred.

The return code of the callback is ignored.

 sub max_4_fields (CSV::Row $r) {
     $r.elems > 4 and $r.splice (4);
     } # max_4_fields

 csv(in => csv(in => "file.csv"), out => $*OUT,
     callbacks => { before print => \&max_4_fields });

 csv(in => csv(in => "file.csv"), out => $*OUT,
     before print => { $^row.elems > 4 and $^row.splice(4) });

This callback is not active for L</combine>.

=back

=head3 Callbacks for csv

The L</csv> allows for some callbacks that do not integrate in internals
but only feature the L</csv> function. XXX: Is this still true?

 csv(in        => "file.csv",
     callbacks => {
         after_parse  => { say "AFTER PARSE";  }, # first
         after_in     => { say "AFTER IN";     }, # second
         on_in        => { say "ON IN";        }, # third
         },
     );

 csv(in        => $aoh,
     out       => "file.csv",
     callbacks => {
         on_in        => { say "ON IN";        }, # first
         before_out   => { say "BEFORE OUT";   }, # second
         before_print => { say "BEFORE PRINT"; }, # third
         },
     );

=over 2

=item filter
X<filter>

This callback can be used to filter records. It is called just after a new
record has been scanned. The callback will be invoked with the current
L</CSV::Row> and should return C<True> for records to accept and C<False>
for records to reject.

 csv (in => "file.csv", filter => {
            $^row[2] ~~ /a/ &&  # third field should contain an "a"
            $^row[4].chars > 4  # length of the 5th field minimal 5
            });

 csv (in => "file.csv", filter => "not_blank");
 csv (in => "file.csv", filter => "not_empty");
 csv (in => "file.csv", filter => "filled");

If the filter is used to I<alter> the content of a field, make sure that
the sub, block or callable returns true in order not to have that record
skipped:

 filter => { $^row[1].text .= uc }

will upper-case the second field, and then skip it if the resulting content
evaluates to false. To always accept, end with truth:

 filter => { $^row[1].text .= uc; 1 }}

If the Callable uses a Hash as prototype, the output is set to Hash and the
C<headers> to C<auto> if unset

 filter => { %^row<foo> ~~ /a/ }

B<Predefined filters>

Given a file like (line numbers prefixed for doc purpose only):

 1:1,2,3
 2:
 3:,
 4:""
 5:,,
 6:, ,
 7:"",
 8:" "
 9:4,5,6

=over 2

=item not_blank
=item not-blank

Filter out the blank lines

This filter is a shortcut for

 filter => { $^row.elems > 1 or
             $^row[0].defined && $^row[0] ne "" or
             $^row[0].is-quoted }

With the given example, line 2 will be skipped.

=item not_empty
=item not-empty

Filter out lines where all the fields are empty.

This filter is a shortcut for

 filter => { $^row.first: { .defined && $_ ne ""   }}

A space is not regarded being empty, so given the example data, lines 2, 3,
4, 5, and 7 are skipped.

=item filled

Filter out lines that have no visible data

This filter is a shortcut for

 filter => { $^row.first: { .defined && $_ ~~ /\S/ }}

This filter rejects all lines that I<not> have at least one field that does
not evaluate to the empty string.

With the given example data, this filter would skip lines 2 through 8.

=back

=item after_in
X<after_in>
X<after-in>

This callback is invoked for each record after all records have been parsed
but before returning the reference to the caller.

This callback can also be passed as an attribute to L</csv> without the
C<callbacks> wrapper.

=item before_out
X<before_out>
X<before-out>

This callback is invoked for each record before the record is printed.

This callback can also be passed as an attribute to L</csv> without the
C<callbacks> wrapper.

=item on_in
X<on_in>
X<on-in>

This callback acts exactly as the L</after_in> or the L</before_out> hooks.

This callback can also be passed as an attribute to L</csv> without the
C<callbacks> wrapper.

=back

=head1 EXAMPLES

=head2 Reading a CSV file line by line:

 my $csv = Text::CSV.new(:auto_diag);
 my $fh = open "file.csv", :r, :!chomp;
 while (my @row = $csv.getline($fh)) {
     # do something with @$row
     }
 $fh.close;

=head3 Reading only a single column

 my $csv = Text::CSV.new(:auto_diag);
 my $fh = open "file.csv", :r, :!chomp;
 # get only the 4th column
 my @column = $csv.getline_all($fh).map(*[3]);
 $fh.close;

with L</csv>, you could do

 my @column = csv(in => "file.csv", fragment => "col=4").map(*[0]);

or

 my @column = csv(in => "file.csv", fragment => "col=4").map(*.flat);

=head2 Parsing CSV strings:

 my $csv = Text::CSV.new(:keep_meta);

 my $sample_input_string =
     q{"I said, ""Hi!""",Yes,"",2.34,,"1.09","\x[20ac]",};
 if ($csv.parse($sample_input_string)) {
     my @field = $csv.fields;
     for ^@field.elems -> $col {
         my $quo = $csv.is_quoted($col) ? $csv.{quote_char} : "";
         printf "%2d: %s%s%s\n", $col, $quo, $field[$col], $quo;
         }
     }
 else {
     print STDERR "parse failed on argument: ",
         $csv.error_input, "\n";
     $csv.error_diag;
     }

=head2 Printing CSV data

=head3 The fast way: using L</print>

An example for creating C<CSV> files using the L</print> method:

 my $csv = Text::CSV.new(eol => $*OUT.nl);
 open my $fh, ">", "foo.csv" or die "foo.csv: $!";
 for 1..10 -> $x {
     $csv.print($fh, [ $x, ~$x ]) or $csv.error_diag;
     }
 close $fh or die "$tbl.csv: $!";

=head3 The slow way: using L</combine> and L</string>

or using the slower L</combine> and L</string> methods:

 my $csv = Text::CSV.new;

 open my $csv_fh, ">", "hello.csv" or die "hello.csv: $!";

 my @sample_input_fields = (
     'You said, "Hello!"',   5.67,
     '"Surely"',   '',   '3.14159');
 if ($csv.combine(@sample_input_fields)) {
     print $csv_fh $csv.string, "\n";
     }
 else {
     print "combine failed on argument: ",
         $csv.error_input, "\n";
     }
 close $csv_fh or die "hello.csv: $!";

=head2 Rewriting CSV

Rewrite C<CSV> files with C<;> as separator character to well-formed C<CSV>:

 use Text::CSV qw( csv );
 csv(in => csv(in => "bad.csv", sep => ";"), out => $*OUT);

=head2 Dumping database tables to CSV

Dumping a database table can be simple as this (TIMTOWTDI):

 my $dbh = DBI.connect(...);
 my $sql = "select * from foo";

 # using your own loop
 open my $fh, ">", "foo.csv" or die "foo.csv: $!\n";
 my $csv = Text::CSV.new(eol => "\r\n");
 my $sth = $dbh.prepare($sql); $sth.execute;
 $csv.print($fh, $sth.{NAME_lc});
 while (my $row = $sth.fetch) {
     $csv.print($fh, $row);
     }

 # using the csv function, all in memory
 csv(out => "foo.csv", in => $dbh.selectall_arrayref($sql));

 # using the csv function, streaming with callbacks
 my $sth = $dbh.prepare($sql); $sth.execute;
 csv(out => "foo.csv", in => { $sth.fetch            });
 csv(out => "foo.csv", in => { $sth.fetchrow_hashref });

Note that this does not discriminate between "empty" values and NULL-values
from the database, as both will be the same empty field in CSV. To enable
distinction between the two, use L<C<quote_empty>|/quote_empty>.

 csv(out => "foo.csv", in => { $sth.fetch }, :quote_empty);

If the database import utility supports special sequences to insert C<NULL>
values into the database, like MySQL/MariaDB supports C<\N>, use a filter
or a map

 csv(out => "foo.csv", in => { $sth.fetch },
                    on_in => { $_ //= "\\N" for @$_[1] }); # WIP

 while (my @row = $sth.fetch) {
     $csv.print($fh, @row.map({ * // "\\N" }));
     }

these special sequences are not recognized by Text::CSV on parsing the
CSV generated like this, but map and filter are your friends again

 while (my @row = $csv.getline($io)) {
     $sth.execute(@row.map({ $_ eq "\\N" ?? Nil !! $_ }));
     }

 csv(in => "foo.csv", filter => { 1 => {
     $sth.execute(@{$_[1]}.map({ $_ eq "\\N" ?? Nil !! $_ }); False; }});

=head2 The examples folder

For more extended examples, see the F<examples/> C<1>) sub-directory in the
original distribution or the git repository C<2>).

 1. https://github.com/Tux/CSV/tree/master/examples/
 2. https://github.com/Tux/CSV/

The following files can be found there:

=over 2

=item csv-check
X<csv-check>

This is a command-line tool to check the C<CSV> file and report on its
content.

TODO

=item csv2xls
X<csv2xls>

A script to convert C<CSV> to Microsoft Excel.

TODO

=item csvdiff
X<csvdiff>

A script that provides colorized diff on sorted CSV files, assuming first
line is header and first field is the key. Output options include colorized
ANSI escape codes or HTML.

TODO

=back

################################################################################

=head1 CAVEATS

=head2 Microsoft Excel

The import/export from Microsoft Excel is a I<risky task>, according to the
documentation in C<Text::CSV::Separator>. Microsoft uses the system's list
separator defined in the regional settings, which happens to be a semicolon
for Dutch, German and Spanish (and probably some others as well). For the
English locale, the default is a comma. In Windows however, the user is
free to choose a predefined locale, and then change I<every> individual
setting in it, so checking the locale is no solution.

A lone first line with just

 sep=;

will be recognized and honored: it will set L<C<sep>|/sep> to C<;> and skip
that line.

=head1 TODO / WIP / NYI

=over 2

=item Real binary data

The solution would be a working C<utf8-c8> encoding.

=item BOM detection

There is no working solution yet for detection of BOM on the L</header>
method. Besides that, not all encodings are supported in raku.

=item on-in and before-print callbacks

The L</on-in> callback currently is an alias for L</after-parse> if the
latter is not specified.

=item Examples

Convert the perl5 example/tool files to raku versions

=item Metadata and CSV for the web

L<Metadata Vocabulary for Tabular Data|http://w3c.github.io/csvw/metadata/>
(a W3C editor's draft) could be an example for supporting more metadata.

W3C's work L<CSV on the Web: Use Cases and
Requirements|http://w3c.github.io/csvw/use-cases-and-requirements/index.html>
is almost finished and worth looking at.

=item Cookbook

Write a document that has recipes for most known non-standard (and maybe
some standard) C<CSV> formats, including formats that use C<TAB>, C<;>,
C<|>, or other non-comma separators.

Examples could be taken from W3C's L<CSV on the Web: Use Cases and
Requirements|http://w3c.github.io/csvw/use-cases-and-requirements/index.html>

=back

=head1 DIAGNOSTICS

Still under construction ...

This section describes the error codes that are used in perl5's module
L<Text::CSV_XS>, and several of these errors are either not applicable in
raku or changed slightly. Once all of the API is finished, this section
will be cleaned up. The intention of the error coded however remains.

If an error occurs, L<C<< $csv.error_diag >>|/error_diag> can be used to
get information on the cause of the failure. Note that for speed reasons
the internal value is never cleared on success, so using the value returned
by L</error_diag> in normal cases - when no error occurred - may cause
unexpected results.

If the constructor failed, the cause will be thrown as an Exception that
represents L</error_diag>.

The C<< $csv.error_diag >> method is automatically invoked upon error when
the contractor was called with L<C<auto_diag>|/auto_diag> set to C<True>.

Errors can be (individually) caught using the L</error> callback.

The errors as described below are available. I have tried to make the error
itself explanatory enough, but more descriptions will be added. For most of
these errors, the first three capitals describe the error category:

=over 2

=item *
INI

Initialization error or option conflict.

=item *
ECR

Carriage-Return related parse error.

=item *
EOF

End-Of-File related parse error.

=item *
EIQ

Parse error inside quotation.

=item *
EIF

Parse error inside field.

=item *
ECB

Combine error.

=item *
EHR

Hash parse related error.

=item *
EHK

Errors related to hooks/callbacks.

=item *
CSV

Errors related to the csv function.

=back

And below should be the complete list of error codes that can be returned:

=over 2

=item *
1001 "INI - separator is equal to quote- or escape sequence"
X<1001>

The L<separation sequence|/sep_char> cannot be equal to L<the quotation
sequence|/quote_char> or to L<the escape sequence|/escape_char>, as this
would invalidate all parsing rules.

=item *
1002 "INI - allow_whitespace with escape_char or quote_char SP or TAB"
X<1002>

Using the L<C<allow_whitespace>|/allow_whitespace> attribute when either
L<C<quote_char>|/quote_char> or L<C<escape_char>|/escape_char> is equal to
C<SPACE> or C<TAB> is too ambiguous to allow.

=item *
1003 "INI - \r or \n in main attr not allowed"
X<1003>

Using default L<C<eol>|/eol> sequences in either
L<separation sequence|/sep_char>, L<quotation sequence|/quote_char>, or
L<escape sequence|/escape_char> is not allowed.

=item *
1004 "INI - callbacks should be undefined or a hashref"
X<1004>

The L<C<callbacks>|/Callbacks> attribute only allows one to be undefined or
a hash reference.

=item *
1010 "INI - the header is empty"
X<1010>

The header line parsed in the L</header> is empty.

=item *
1011 "INI - the header contains more than one valid separator"
X<1011>

The header line parsed in the L</header> contains more than one (unique)
separator character out of the allowed set of separators.

=item *
1012 "INI - the header contains an empty field"
X<1012>

The header line parsed in the L</header> is contains an empty field.

=item *
1013 "INI - the header contains nun-unique fields"
X<1013>

The header line parsed in the L</header> contains at least two identical
fields.

=item *
2010 "ECR - QUO char inside quotes followed by CR not part of EOL"
X<2010>

When L<C<eol>|/eol> has been set to anything but the default, like
C<"\r\t\n">, and the C<"\r"> is following the B<second> (closing)
L<C<quote_char>|/quote_char>, where the characters following the C<"\r"> do
not make up the L<C<eol>|/eol> sequence, this is an error.

=item *
2011 "ECR - Characters after end of quoted field"
X<2011>

Sequences like C<1,foo,"bar"baz,22,1> are not allowed. C<"bar"> is a quoted
field and after the closing double-quote, there should be either a new-line
sequence or a separation sequence.

=item *
2012 "EOF - End of data in parsing input stream"
X<2012>

Self-explaining. End-of-file while inside parsing a stream. Can happen only
when reading from streams with L</getline>, as using L</parse> is done on
strings that are not required to have a trailing L<C<eol>|/eol>.

=item *
2013 "INI - Specification error for fragments RFC7111"
X<2013>

Invalid specification for URI L</fragment> specification.

=item *
2021 "EIQ - NL char inside quotes, binary off"
X<2021>

Sequences like C<1,"foo\nbar",22,1> are allowed only when the binary option
has been selected with the constructor.

=item *
2022 "EIQ - CR char inside quotes, binary off"
X<2022>

Sequences like C<1,"foo\rbar",22,1> are allowed only when the binary option
has been selected with the constructor.

=item *
2023 "EIQ - QUO sequence not allowed"
X<2023>

Sequences like C<"foo "bar" baz",qu> and C<2023,",2008-04-05,"Foo, Bar",\n>
will cause this error.

=item *
2024 "EIQ - EOF cannot be escaped, not even inside quotes"
X<2024>

The escape sequence is not allowed as last item in an input stream.

=item *
2025 "EIQ - Loose unescaped escape"
X<2025>

An escape sequence should escape only characters that need escaping.

Allowing the escape for other characters is possible with the attribute
L</allow_loose_escapes>.

=item *
2026 "EIQ - Binary character inside quoted field, binary off"
X<2026>

Binary characters are not allowed by default. Exceptions are fields that
contain valid UTF-8, that will automatically be upgraded if the content is
valid UTF-8. Set L<C<binary>|/binary> to C<1> to accept binary data.

=item *
2027 "EIQ - Quoted field not terminated"
X<2027>

When parsing a field that started with a quotation sequence, the field is
expected to be closed with a quotation sequence. When the parsed line is
exhausted before the quote is found, that field is not terminated.

=item *
2030 "EIF - NL char inside unquoted verbatim, binary off"
X<2030>

=item *
2031 "EIF - CR char is first char of field, not part of EOL"
X<2031>

=item *
2032 "EIF - CR char inside unquoted, not part of EOL"
X<2032>

=item *
2034 "EIF - Loose unescaped quote"
X<2034>

=item *
2035 "EIF - Escaped EOF in unquoted field"
X<2035>

=item *
2036 "EIF - ESC error"
X<2036>

=item *
2037 "EIF - Binary character in unquoted field, binary off"
X<2037>

=item *
2110 "ECB - Binary character in Combine, binary off"
X<2110>

=item *
2200 "EIO - print to IO failed. See errno"
X<2200>

=item *
3001 "EHR - Unsupported syntax for column_names"
X<3001>

=item *
3002 "EHR - getline_hr called before column_names"
X<3002>

=item *
3003 "EHR - bind_columns and column_names fields count mismatch"
X<3003>

=item *
3004 "EHR - bind_columns only accepts refs to scalars"
X<3004>

=item *
3006 "EHR - bind_columns did not pass enough refs for parsed fields"
X<3006>

=item *
3007 "EHR - bind_columns needs refs to writable scalars"
X<3007>

=item *
3008 "EHR - unexpected error in bound fields"
X<3008>

=item *
3009 "EHR - print_hr called before column_names"
X<3009>

=item *
3010 "EHR - print_hr called with invalid arguments"
X<3010>

=item *
3100 "EHK - Unsupported callback"
X<3100>

=item *
4001 "PRM - The key does not exist as field in the data",
X<4001>

You cannot set the key from a non-existing column.

If you were using a key list, I<all> keys should exist.

=item *
5000 "CSV - Unsupported type for in"
X<5000>

=item *
5001 "CSV - Unsupported type for out"
X<5001>

=back

=head1 SEE ALSO

Modules in perl5:

L<IO::File>, L<IO::Handle>, L<IO::Wrap>, L<Text::CSV_XS>, L<Text::CSV>,
L<Text::CSV_PP>, L<Text::CSV::Encoded>, L<Text::CSV::Separator>,
L<Text::CSV::Slurp>, L<Spreadsheet::CSV> and L<Spreadsheet::Read>;

=head1 AUTHOR

H.Merijn Brand F<E<lt>hmbrand@cpan.orgE<gt>> wrote this based on the
features provided by perl5's Text::CSV_XS.

Liz Mattijsen helped in getting the best out of raku.

=head1 COPYRIGHT AND LICENSE

 Copyright (C) 2014-2023 H.Merijn Brand. All rights reserved.

This library is free software; you can redistribute and/or modify it under
the same terms as Perl itself.

=cut

=for elvis
:ex:se gw=75|color guide #ff0000:

=end pod


