From d85e0bcba9ac7d8f25c41b6ffaefb86af0809267 Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Fri, 10 Jul 2026 22:05:32 -0700 Subject: [PATCH 1/4] feat: Clean up public API and hide internals --- CHANGELOG.md | 13 + app/LazyBenchmark.hs | 44 +-- app/Synthesis.hs | 7 +- cabal.project.bare | 1 + dataframe-arrow/dataframe-arrow.cabal | 9 +- dataframe-arrow/ffi-export/DataFrame/FFI.hs | 73 ++--- dataframe-core/dataframe-core.cabal | 28 +- .../DataFrame/Display/Terminal/Colours.hs | 0 .../DataFrame/Display/Terminal/PrettyPrint.hs | 14 +- .../{src => src-internal}/DataFrame/Errors.hs | 0 .../DataFrame/Internal/AggKernel.hs | 50 +--- .../DataFrame/Internal/AggKernelDirect.hs | 51 +--- .../DataFrame/Internal/AggKernelPar.hs | 69 +---- .../DataFrame/Internal/AggPlan.hs | 45 +-- .../DataFrame/Internal/Column.hs | 186 +++--------- .../DataFrame/Internal/ColumnBuilder.hs | 0 .../DataFrame/Internal/ColumnMerge.hs | 19 +- .../DataFrame/Internal/DataFrame.hs | 34 +-- .../DataFrame/Internal/DictEncode.hs | 62 ++-- .../DataFrame/Internal/Expression.hs | 45 +-- .../DataFrame/Internal/Grouping.hs | 88 ++---- .../DataFrame/Internal/GroupingDirect.hs | 36 +-- .../DataFrame/Internal/GroupingPar.hs | 69 +---- .../DataFrame/Internal/Hash.hs | 39 +-- .../DataFrame/Internal/HashTable.hs | 31 +- .../DataFrame/Internal/Interpreter.hs | 76 +---- .../DataFrame/Internal/Nullable.hs | 63 +---- .../DataFrame/Internal/PackedText.hs | 46 +-- .../DataFrame/Internal/ParRadixSort.hs | 28 +- .../DataFrame/Internal/Pretty.hs | 18 +- .../DataFrame/Internal/RadixRank.hs | 23 +- .../DataFrame/Internal/Row.hs | 48 +--- .../DataFrame/Internal/RowHash.hs | 36 +-- .../DataFrame/Internal/Simplify.hs | 5 - .../DataFrame/Internal/Types.hs | 6 +- .../DataFrame/Internal/Utf8.hs | 7 +- .../DataFrame/Operators.hs | 0 dataframe-core/src/DataFrame/Core.hs | 104 +++++++ dataframe-csv-th/dataframe-csv-th.cabal | 6 +- dataframe-csv/dataframe-csv.cabal | 28 +- .../DataFrame/IO/Internal/MutableColumn.hs | 0 dataframe-csv/src/DataFrame/IO/CSV.hs | 16 +- .../src/DataFrame/IO/CSV/Internal/Options.hs | 2 +- .../src/DataFrame/IO/CSV/Internal/Read.hs | 15 +- .../dataframe-expr-serializer.cabal | 8 +- dataframe-fastcsv/dataframe-fastcsv.cabal | 22 +- .../src/DataFrame/IO/CSV/Fast.hs | 25 +- .../src/DataFrame/IO/CSV/Fast/Columns.hs | 11 +- .../tests/Operations/ChunkParallel.hs | 15 +- dataframe-fastcsv/tests/Operations/ReadCsv.hs | 26 +- .../tests/Operations/TypedExtraction.hs | 20 +- dataframe-fusion/dataframe-fusion.cabal | 15 +- .../src/DataFrame/Fusion/Typed.hs | 26 +- dataframe-hasktorch/dataframe-hasktorch.cabal | 4 +- .../src/DataFrame/Hasktorch.hs | 71 +---- .../dataframe-huggingface.cabal | 10 +- .../src/DataFrame/IO/HuggingFace.hs | 26 +- dataframe-json/dataframe-json.cabal | 6 +- dataframe-lazy/dataframe-lazy.cabal | 22 +- dataframe-lazy/src/DataFrame/Lazy.hs | 6 +- dataframe-lazy/src/DataFrame/Lazy/IO/CSV.hs | 44 +-- .../src/DataFrame/Lazy/Internal/DataFrame.hs | 12 +- .../src/DataFrame/Lazy/Internal/Executor.hs | 155 ++-------- .../DataFrame/Lazy/Internal/LogicalPlan.hs | 2 +- .../src/DataFrame/Lazy/Internal/Optimizer.hs | 36 +-- .../DataFrame/Lazy/Internal/PhysicalPlan.hs | 2 +- dataframe-lazy/src/DataFrame/Typed/Lazy.hs | 11 +- dataframe-learn/README.md | 18 +- dataframe-learn/dataframe-learn.cabal | 91 +++++- .../DataFrame/DecisionTree/Cart.hs | 6 +- .../DataFrame/DecisionTree/Categorical.hs | 3 +- .../DataFrame/DecisionTree/CondVec.hs | 0 .../DataFrame/DecisionTree/Fit.hs | 2 +- .../DataFrame/DecisionTree/Linear.hs | 0 .../DataFrame/DecisionTree/Numeric.hs | 6 +- .../DataFrame/DecisionTree/Pool.hs | 0 .../DataFrame/DecisionTree/Predict.hs | 0 .../DataFrame/DecisionTree/Prune.hs | 0 .../DataFrame/DecisionTree/Tao.hs | 2 +- .../DataFrame/DecisionTree/Types.hs | 0 .../DataFrame/Featurize/Internal.hs | 3 +- .../DataFrame/LinearAlgebra.hs | 5 +- .../DataFrame/LinearAlgebra/Eigen.hs | 5 +- .../DataFrame/LinearAlgebra/Solve.hs | 0 .../DataFrame/LinearSolver.hs | 67 ++--- .../DataFrame/LinearSolver/Loss.hs | 0 .../{src => src-internal}/DataFrame/Random.hs | 6 +- .../DataFrame/SymbolicRegression/Expr.hs | 7 +- .../DataFrame/SymbolicRegression/GP.hs | 7 +- .../DataFrame/SymbolicRegression/Optimize.hs | 3 +- .../DataFrame/SymbolicRegression/Simplify.hs | 6 +- dataframe-learn/src/DataFrame/DecisionTree.hs | 74 +++-- .../src/DataFrame/DecisionTree/Regression.hs | 12 +- dataframe-learn/src/DataFrame/Learn.hs | 51 ++++ dataframe-learn/src/DataFrame/LinearModel.hs | 10 + .../src/DataFrame/LinearModel/Regression.hs | 8 +- dataframe-learn/src/DataFrame/SVM.hs | 9 +- dataframe-learn/src/DataFrame/Segmented.hs | 54 ++-- .../tests-internal}/Cart.hs | 30 +- .../tests-internal/DataFrameApi.hs | 22 ++ .../tests-internal}/DecisionTree.hs | 150 +++++----- .../tests-internal}/Learn/EdgeCases.hs | 128 +++------ .../tests-internal}/Learn/NumericalRigor.hs | 72 ++--- .../tests-internal}/Learn/Numerics.hs | 0 .../tests-internal}/Learn/Symbolic.hs | 12 +- .../tests-internal}/LinearSolver.hs | 102 ++----- dataframe-learn/tests-internal/Main.hs | 48 ++++ .../tests-internal}/Properties/Simplify.hs | 17 +- .../tests-internal}/TreePruning.hs | 3 +- .../tests-internal}/Worklist.hs | 105 ++----- .../dataframe-operations.cabal | 23 +- .../DataFrame/Internal/Statistics.hs | 0 dataframe-operations/src/DataFrame/Monad.hs | 30 +- .../src/DataFrame/Operations/Core.hs | 114 ++++---- .../src/DataFrame/Operations/Join.hs | 179 +++++------- .../src/DataFrame/Operations/Merge.hs | 9 +- .../src/DataFrame/Operations/Permutation.hs | 10 +- .../src/DataFrame/Operations/Statistics.hs | 29 +- .../src/DataFrame/Operations/Subset.hs | 46 ++- .../DataFrame/Operations/Transformations.hs | 21 +- .../src/DataFrame/Operations/Typing.hs | 88 ++---- .../dataframe-parquet-th.cabal | 10 +- dataframe-parquet/dataframe-parquet.cabal | 11 +- dataframe-parsing/dataframe-parsing.cabal | 25 +- .../DataFrame/Internal/Binary.hs | 0 .../DataFrame/Internal/Parsing.hs | 0 .../DataFrame/Internal/Parsing/Fast.hs | 13 +- .../DataFrame/Internal/Parsing/Fast/Common.hs | 17 +- .../DataFrame/Internal/Parsing/Fast/Double.hs | 21 +- .../DataFrame/Internal/Parsing/Fast/Int.hs | 5 +- .../DataFrame/Internal/Parsing/Fast/Token.hs | 28 +- dataframe-parsing/src/DataFrame/Schema.hs | 9 + .../dataframe-persistent.cabal | 11 +- dataframe-th/dataframe-th.cabal | 15 +- .../src/DataFrame/Internal/Schema/TH.hs | 19 +- .../src/DataFrame/Typed/TH/Records.hs | 9 + dataframe-viz/dataframe-viz.cabal | 7 +- .../docs/base_scripts/base_readme.md | 3 +- .../DataFrame/Display/Internal/VegaLite.hs | 25 +- .../src/DataFrame/Display/Web/Plot.hs | 20 +- dataframe.cabal | 130 ++++----- docs/base/haskell_for_data_analysis.md | 2 +- docs/base_scripts/base_readme.md | 2 +- docs/coming_from_other_implementations.md | 6 +- docs/working_with_nulls.md | 2 +- examples/OneBillionRowChallenge.hs | 2 +- examples/examples.cabal | 5 + ffi/DataFrame/IR.hs | 7 +- scratch-staged.project | 18 ++ scripts/release.sh | 4 +- src/DataFrame.hs | 267 +++--------------- tests/IO/CsvGolden.hs | 16 +- tests/LazyParity.hs | 22 +- tests/LazyParquet.hs | 4 +- tests/Learn/Ensembles.hs | 1 - tests/Learn/MetricsTests.hs | 5 - tests/Learn/Models.hs | 2 - tests/Learn/SklearnParity.hs | 8 +- tests/Main.hs | 26 +- tests/Operations/Record.hs | 11 +- 160 files changed, 1771 insertions(+), 2825 deletions(-) rename dataframe-core/{src => src-internal}/DataFrame/Display/Terminal/Colours.hs (100%) rename dataframe-core/{src => src-internal}/DataFrame/Display/Terminal/PrettyPrint.hs (86%) rename dataframe-core/{src => src-internal}/DataFrame/Errors.hs (100%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/AggKernel.hs (77%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/AggKernelDirect.hs (82%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/AggKernelPar.hs (78%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/AggPlan.hs (83%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Column.hs (90%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/ColumnBuilder.hs (100%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/ColumnMerge.hs (92%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/DataFrame.hs (91%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/DictEncode.hs (60%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Expression.hs (89%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Grouping.hs (75%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/GroupingDirect.hs (81%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/GroupingPar.hs (72%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Hash.hs (64%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/HashTable.hs (63%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Interpreter.hs (92%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Nullable.hs (87%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/PackedText.hs (70%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/ParRadixSort.hs (86%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Pretty.hs (80%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/RadixRank.hs (77%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Row.hs (79%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/RowHash.hs (79%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Simplify.hs (97%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Types.hs (96%) rename dataframe-core/{src => src-internal}/DataFrame/Internal/Utf8.hs (90%) rename dataframe-core/{src => src-internal}/DataFrame/Operators.hs (100%) create mode 100644 dataframe-core/src/DataFrame/Core.hs rename dataframe-csv/{src => src-internal}/DataFrame/IO/Internal/MutableColumn.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Cart.hs (97%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Categorical.hs (99%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/CondVec.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Fit.hs (99%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Linear.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Numeric.hs (98%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Pool.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Predict.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Prune.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Tao.hs (99%) rename dataframe-learn/{src => src-internal}/DataFrame/DecisionTree/Types.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/Featurize/Internal.hs (98%) rename dataframe-learn/{src => src-internal}/DataFrame/LinearAlgebra.hs (94%) rename dataframe-learn/{src => src-internal}/DataFrame/LinearAlgebra/Eigen.hs (96%) rename dataframe-learn/{src => src-internal}/DataFrame/LinearAlgebra/Solve.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/LinearSolver.hs (83%) rename dataframe-learn/{src => src-internal}/DataFrame/LinearSolver/Loss.hs (100%) rename dataframe-learn/{src => src-internal}/DataFrame/Random.hs (93%) rename dataframe-learn/{src => src-internal}/DataFrame/SymbolicRegression/Expr.hs (93%) rename dataframe-learn/{src => src-internal}/DataFrame/SymbolicRegression/GP.hs (96%) rename dataframe-learn/{src => src-internal}/DataFrame/SymbolicRegression/Optimize.hs (96%) rename dataframe-learn/{src => src-internal}/DataFrame/SymbolicRegression/Simplify.hs (87%) create mode 100644 dataframe-learn/src/DataFrame/Learn.hs rename {tests => dataframe-learn/tests-internal}/Cart.hs (75%) create mode 100644 dataframe-learn/tests-internal/DataFrameApi.hs rename {tests => dataframe-learn/tests-internal}/DecisionTree.hs (91%) rename {tests => dataframe-learn/tests-internal}/Learn/EdgeCases.hs (75%) rename {tests => dataframe-learn/tests-internal}/Learn/NumericalRigor.hs (82%) rename {tests => dataframe-learn/tests-internal}/Learn/Numerics.hs (100%) rename {tests => dataframe-learn/tests-internal}/Learn/Symbolic.hs (93%) rename {tests => dataframe-learn/tests-internal}/LinearSolver.hs (86%) create mode 100644 dataframe-learn/tests-internal/Main.hs rename {tests => dataframe-learn/tests-internal}/Properties/Simplify.hs (90%) rename {tests => dataframe-learn/tests-internal}/TreePruning.hs (97%) rename {tests => dataframe-learn/tests-internal}/Worklist.hs (79%) rename dataframe-operations/{src => src-internal}/DataFrame/Internal/Statistics.hs (100%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Binary.hs (100%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Parsing.hs (100%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Parsing/Fast.hs (83%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Parsing/Fast/Common.hs (81%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Parsing/Fast/Double.hs (84%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Parsing/Fast/Int.hs (92%) rename dataframe-parsing/{src => src-internal}/DataFrame/Internal/Parsing/Fast/Token.hs (85%) create mode 100644 dataframe-parsing/src/DataFrame/Schema.hs create mode 100644 scratch-staged.project diff --git a/CHANGELOG.md b/CHANGELOG.md index 59492600..3ff5e522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Revision history for dataframe +## 3.0.0.0 + +A breaking release that removes internal functions from the public API. + +### Highlights +* Internalish modules now have public APIs/contracts. `DataFrame.Core`, `DataFrame.Schema`, and `DataFrame.Learn`. +* `import DataFrame` now surfaces `Columnable`, `Columnable'`, `eSize`, `NamedExpr`, + `SchemaType(..)`, and the `Semigroup`/`Monoid DataFrame` instances directly. + +### Breaking changes +* Internal modules are no longer on the public surface. +* Subpackage major bumps. + ## 2.3.0.0 ### Breaking changes diff --git a/app/LazyBenchmark.hs b/app/LazyBenchmark.hs index 7b67e9a4..9cfd43f9 100644 --- a/app/LazyBenchmark.hs +++ b/app/LazyBenchmark.hs @@ -2,21 +2,14 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | End-to-end smoke test and benchmark for the Lazy streaming API. +{- | End-to-end smoke test and benchmark for the Lazy streaming API: + generate a CSV then run five Lazy queries over it. Usage: - cabal run lazy-bench [-- [OPTIONS]] - - Options: - --rows N Number of rows to generate (default: 1_000_000_000) - --file PATH Output CSV path (default: /tmp/lazy_1b.csv) - --skip-gen Skip generation if the file already exists - - The executable generates a CSV file (streaming, constant memory) then - runs five Lazy queries over it, printing timing and result summaries. - - For heap/GC stats run with: - cabal run lazy-bench -- +RTS -s -RTS + cabal run lazy-bench [-- [OPTIONS]] (+RTS -s -RTS for heap stats) + --rows N rows to generate (default 1_000_000_000) + --file PATH output CSV path (default /tmp/lazy_1b.csv) + --skip-gen reuse the file if it already exists -} module Main where @@ -26,9 +19,9 @@ import qualified Data.Map as M import qualified Data.Text as T import Data.Time (UTCTime, diffUTCTime, getCurrentTime) import qualified DataFrame as D -import DataFrame.Internal.Schema (Schema (..), schemaType) import qualified DataFrame.Lazy as L import DataFrame.Operators +import DataFrame.Schema (Schema (..), schemaType) import System.Directory (doesFileExist, getFileSize) import System.Environment (getArgs) import System.Exit (exitFailure) @@ -185,9 +178,6 @@ main = do n = optRows opts pathT = T.pack path - -- ----------------------------------------------------------------------- - -- Phase 1: Generate - -- ----------------------------------------------------------------------- putStrLn "=== Lazy API 1B-row benchmark ===" putStrLn $ " rows : " ++ commas n putStrLn $ " file : " ++ path @@ -205,12 +195,8 @@ main = do sz <- getFileSize path putStrLn $ " file size: " ++ showBytes sz - -- ----------------------------------------------------------------------- - -- Phase 2: Lazy queries - -- ----------------------------------------------------------------------- putStrLn "\nPhase 2: Lazy queries" - -- Schema for the generated CSV: id (Int), x (Double), y (Double), category (Text) let schema = Schema $ M.fromList @@ -220,25 +206,17 @@ main = do , ("category", schemaType @T.Text) ] - -- Q1: Preview — limit 20, no filter. - -- Demonstrates that the executor reads only the first batch. runQuery "Q1 — preview first 20 rows (no filter)" $ L.runDataFrame $ L.take 20 $ L.scanCsv schema pathT - -- Q2: Filter + limit. - -- x > 0.999 ≈ 0.1% of rows. With a 512K-row batch the executor finds - -- ~512 matches in the first batch and stops — reads only one batch. runQuery "Q2 — filter (x > 0.999), limit 20" $ L.runDataFrame $ L.take 20 $ L.filter (col @Double "x" .> lit (0.999 :: Double)) $ L.scanCsv schema pathT - -- Q3: Filter + derive + select + limit. - -- Shows projection pushdown: only id/x/y/category are read, z is derived. - -- Predicate pushdown moves the filter into the scan batch loop. runQuery "Q3 — filter (x > 0.999), derive z = x*y, select [id,z], limit 20" $ L.runDataFrame $ L.take 20 $ @@ -247,10 +225,6 @@ main = do L.filter (col @Double "x" .> lit (0.999 :: Double)) $ L.scanCsv schema pathT - -- Q4: Filter fusion demo. - -- Two consecutive filters are fused into one AND predicate by the optimizer. - -- Result: rows where x > 0.5 AND y > 0.5 (≈ 25% of total). - -- We limit to keep result size manageable. runQuery "Q4 — filter fusion: (x > 0.5) . (y > 0.5), limit 20" $ L.runDataFrame $ L.take 20 $ @@ -258,10 +232,6 @@ main = do L.filter (col @Double "x" .> lit (0.5 :: Double)) $ L.scanCsv schema pathT - -- Q5: Full scan, heavy filter, count results. - -- x > 0.999 across the whole file ≈ 0.1% × N rows. - -- For 1B rows that is ~1M results — materialised into one DataFrame. - -- This query exercises streaming across all batches. runQuery ( "Q5 — full scan, filter (x > 0.999), count (~" ++ approx (n `div` 1000) diff --git a/app/Synthesis.hs b/app/Synthesis.hs index b9361fbd..9a3a144b 100644 --- a/app/Synthesis.hs +++ b/app/Synthesis.hs @@ -47,8 +47,8 @@ main = do D.randomSplit (mkStdGen 4232) 0.7 (DT.thaw (clean train)) testDf = DT.thaw (clean test) - model = - fitDecisionTree + fitted = + fit ( defaultTreeConfig { maxTreeDepth = 5 , minSamplesSplit = 5 @@ -68,8 +68,9 @@ main = do ) (F.fromMaybe 0 (F.col @(Maybe Int) "Survived")) (trainDf |> D.exclude ["PassengerId"]) + model = predict fitted - print model + print fitted putStrLn "Training accuracy: " print $ computeAccuracy (trainDf |> D.derive (F.name prediction) model) diff --git a/cabal.project.bare b/cabal.project.bare index d39d8b44..54190278 100644 --- a/cabal.project.bare +++ b/cabal.project.bare @@ -20,3 +20,4 @@ packages: dataframe-viz dataframe-learn dataframe-lazy + dataframe-expr-serializer diff --git a/dataframe-arrow/dataframe-arrow.cabal b/dataframe-arrow/dataframe-arrow.cabal index a33a8914..0281b9f3 100644 --- a/dataframe-arrow/dataframe-arrow.cabal +++ b/dataframe-arrow/dataframe-arrow.cabal @@ -56,10 +56,11 @@ foreign-library dataframe-arrow other-modules: DataFrame.FFI build-depends: base >= 4 && < 5, - dataframe-core ^>= 1.1, - dataframe-learn ^>= 1.1, - dataframe:arrow-bridge >= 1 && < 3, - dataframe-fastcsv ^>= 1.1.1, + dataframe-core ^>= 2.0, + dataframe-core:internal ^>= 2.0, + dataframe-learn ^>= 2.0, + dataframe:arrow-bridge >= 1 && < 4, + dataframe-fastcsv ^>= 1.2, text >= 2.1 && < 3, aeson >= 0.11 && < 3, bytestring >= 0.11 && < 0.13, diff --git a/dataframe-arrow/ffi-export/DataFrame/FFI.hs b/dataframe-arrow/ffi-export/DataFrame/FFI.hs index eadc080d..80ee2fef 100644 --- a/dataframe-arrow/ffi-export/DataFrame/FFI.hs +++ b/dataframe-arrow/ffi-export/DataFrame/FFI.hs @@ -36,7 +36,8 @@ import DataFrame.DecisionTree ( defaultColumnOrdering, defaultSynthConfig, defaultTreeConfig, - fitDecisionTree, + fit, + predict, ) import DataFrame.IO.Arrow (dataframeToArrow) import DataFrame.IO.CSV.Fast (fastReadCsvWithSchema) @@ -49,14 +50,9 @@ import DataFrame.Internal.Expression (Expr (Col)) foreign export ccall "dfExecutePlan" dfExecutePlan :: CString -> Ptr Word64 -> Ptr Word64 -> IO CInt -{- | Execute a JSON-encoded query plan and return Arrow C Data Interface - pointers to Python via output parameters. - - @planCS@ – JSON plan string (see DataFrame.IR for schema) - @schemaOut@ – receives the ArrowSchema* address as a Word64 - @arrayOut@ – receives the ArrowArray* address as a Word64 - - Returns 0 on success, -1 on error. +{- | Execute a JSON-encoded query plan, returning Arrow C Data Interface +pointers to Python via the @schemaOut@/@arrayOut@ output parameters (each an +address as a 'Word64'). Returns 0 on success, -1 on error. -} dfExecutePlan :: CString -> Ptr Word64 -> Ptr Word64 -> IO CInt dfExecutePlan planCS schemaOut arrayOut = do @@ -88,22 +84,9 @@ foreign export ccall "dfFitDecisionTree" foreign export ccall "dfFreeModel" dfFreeModel :: Ptr CChar -> IO () -{- | Fit a TAO decision tree on the materialized result of @plan_json@, predicting - @target_col@ from the remaining columns. - - The caller owns the returned buffer and must release it with 'dfFreeModel'. - - @target_type@ may be the literal string @"auto"@ to infer the target's type - from the materialized DataFrame's schema, or one of the wire-format type - tags (@"int"@, @"double"@, @"text"@, @"bool"@, …). - - @config_json@ is a JSON object with the serializable TreeConfig fields: - @max_depth@, @min_samples_split@, @min_leaf_size@, @percentiles@, - @expression_pairs@, @tao_iterations@, @tao_convergence_tol@, - @max_expr_depth@, @bool_expansion@, @complexity_penalty@, - @enable_string_ops@, @enable_cross_cols@, @enable_arith_ops@. - - Returns 0 on success, -1 on error. +{- | Fit a TAO decision tree on a plan's materialized result, predicting the +target column from the rest (@target_type@: @"auto"@ or a wire type tag; +@config_json@: 'TreeConfig' fields). Caller frees via 'dfFreeModel'; 0 ok, -1 err. -} dfFitDecisionTree :: CString -> @@ -255,31 +238,31 @@ inferTargetType target df = dispatchType (columnTypeRep (unsafeGetColumn target fitTreeWithType :: T.Text -> TreeConfig -> T.Text -> DataFrame -> IO BS.ByteString fitTreeWithType ttag cfg target df = case ttag of - "int" -> fit @Int - "int8" -> fit @Int8 - "int16" -> fit @Int16 - "int32" -> fit @Int32 - "int64" -> fit @Int64 - "word" -> fit @Word - "word8" -> fit @Word8 - "word16" -> fit @Word16 - "word32" -> fit @Word32 - "word64" -> fit @Word64 - "integer" -> fit @Integer - "double" -> fit @Double - "float" -> fit @Float - "bool" -> fit @Bool - "char" -> fit @Char - "text" -> fit @T.Text - "string" -> fit @String + "int" -> fitEncode @Int + "int8" -> fitEncode @Int8 + "int16" -> fitEncode @Int16 + "int32" -> fitEncode @Int32 + "int64" -> fitEncode @Int64 + "word" -> fitEncode @Word + "word8" -> fitEncode @Word8 + "word16" -> fitEncode @Word16 + "word32" -> fitEncode @Word32 + "word64" -> fitEncode @Word64 + "integer" -> fitEncode @Integer + "double" -> fitEncode @Double + "float" -> fitEncode @Float + "bool" -> fitEncode @Bool + "char" -> fitEncode @Char + "text" -> fitEncode @T.Text + "string" -> fitEncode @String other -> ioError . userError $ "DataFrame.FFI.fitTreeWithType: unsupported target type tag: " ++ T.unpack other where - fit :: forall a. (Columnable a, Ord a) => IO BS.ByteString - fit = do - let expr = fitDecisionTree @a cfg (Col @a target) df + fitEncode :: forall a. (Columnable a, Ord a) => IO BS.ByteString + fitEncode = do + let expr = predict (fit cfg (Col @a target) df) case encodeExprToBytes expr of Right bs -> return bs Left err -> ioError (userError $ "encodeExprToBytes: " ++ err) diff --git a/dataframe-core/dataframe-core.cabal b/dataframe-core/dataframe-core.cabal index ed47d1b1..7821cd2f 100644 --- a/dataframe-core/dataframe-core.cabal +++ b/dataframe-core/dataframe-core.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-core -version: 1.1.1.0 +version: 2.0.0.0 synopsis: Core data structures for the dataframe library. description: Minimal interchange-format types for the @dataframe@ ecosystem: @@ -28,8 +28,9 @@ common warnings -Wunused-local-binds -Wunused-packages -library +library internal import: warnings + visibility: public exposed-modules: DataFrame.Errors DataFrame.Operators @@ -61,17 +62,30 @@ library DataFrame.Internal.Simplify DataFrame.Internal.Types DataFrame.Internal.Utf8 + build-depends: base >= 4 && < 5, + containers >= 0.6.7 && < 0.9, + primitive >= 0.7 && < 0.10, + random >= 1 && < 2, + text >= 2.1 && < 3, + vector ^>= 0.13 + hs-source-dirs: src-internal + default-language: Haskell2010 + +library + import: warnings + exposed-modules: + DataFrame.Core DataFrame.Typed.Freeze DataFrame.Typed.Generic DataFrame.Typed.Record DataFrame.Typed.Schema DataFrame.Typed.Types DataFrame.Typed.Util + reexported-modules: DataFrame.Errors, + DataFrame.Operators build-depends: base >= 4 && < 5, - containers >= 0.6.7 && < 0.9, - primitive >= 0.7 && < 0.10, - random >= 1 && < 2, text >= 2.1 && < 3, - vector ^>= 0.13 + vector ^>= 0.13, + dataframe-core:internal hs-source-dirs: src default-language: Haskell2010 diff --git a/dataframe-core/src/DataFrame/Display/Terminal/Colours.hs b/dataframe-core/src-internal/DataFrame/Display/Terminal/Colours.hs similarity index 100% rename from dataframe-core/src/DataFrame/Display/Terminal/Colours.hs rename to dataframe-core/src-internal/DataFrame/Display/Terminal/Colours.hs diff --git a/dataframe-core/src/DataFrame/Display/Terminal/PrettyPrint.hs b/dataframe-core/src-internal/DataFrame/Display/Terminal/PrettyPrint.hs similarity index 86% rename from dataframe-core/src/DataFrame/Display/Terminal/PrettyPrint.hs rename to dataframe-core/src-internal/DataFrame/Display/Terminal/PrettyPrint.hs index 3afafa0f..6e2341a6 100644 --- a/dataframe-core/src/DataFrame/Display/Terminal/PrettyPrint.hs +++ b/dataframe-core/src-internal/DataFrame/Display/Terminal/PrettyPrint.hs @@ -14,19 +14,18 @@ data RenderFormat = Plain | Markdown -- Utility functions to show a DataFrame as a Markdown-ish table. --- Adapted from: https://stackoverflow.com/questions/5929377/format-list-output-in-haskell --- a type for fill functions +-- Fill functions, adapted from: +-- https://stackoverflow.com/questions/5929377/format-list-output-in-haskell type Filler = Int -> T.Text -> T.Text --- a type for describing table columns +-- A description of one table column. data ColDesc t = ColDesc { colTitleFill :: Filler , colTitle :: T.Text , colValueFill :: Filler } --- functions that fill a string (s) to a given width (n) by adding pad --- character (c) to align left, right, or center +-- Fill text to a given width with a pad character, aligning left, right, or center. fillLeft :: Char -> Int -> T.Text -> T.Text fillLeft c n s = s <> T.replicate (n - T.length s) (T.singleton c) @@ -41,7 +40,7 @@ fillCenter c n s = l = x `div` 2 r = x - l --- functions that fill with spaces +-- Fill with spaces. left :: Int -> T.Text -> T.Text left = fillLeft ' ' @@ -63,9 +62,6 @@ showTable :: T.Text showTable fmt header types columns = let isMarkdown = fmt == Markdown - -- In a GitHub pipe table a literal '|' ends the cell and a newline ends - -- the row, so a value like the operator name <|> would split the row into - -- the wrong columns. Escape both for the Markdown format only. esc = if isMarkdown then escapeMarkdownCell else id hdr = map esc header tys = map esc types diff --git a/dataframe-core/src/DataFrame/Errors.hs b/dataframe-core/src-internal/DataFrame/Errors.hs similarity index 100% rename from dataframe-core/src/DataFrame/Errors.hs rename to dataframe-core/src-internal/DataFrame/Errors.hs diff --git a/dataframe-core/src/DataFrame/Internal/AggKernel.hs b/dataframe-core/src-internal/DataFrame/Internal/AggKernel.hs similarity index 77% rename from dataframe-core/src/DataFrame/Internal/AggKernel.hs rename to dataframe-core/src-internal/DataFrame/Internal/AggKernel.hs index d25ac61b..185df392 100644 --- a/dataframe-core/src/DataFrame/Internal/AggKernel.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/AggKernel.hs @@ -6,25 +6,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Vectorized scatter-accumulate aggregation kernel. - -The grouping layer ('DataFrame.Internal.Grouping') hands us a dense -@rowToGroup@ vector (group id per row, canonical order) plus the number of -groups. For the common reductions this kernel replaces the per-group boxed -expression-interpreter fold with a single unboxed linear pass that scatters -each row's value into primitive per-group accumulator arrays indexed by the -group id. No boxed accumulator record, no per-element dictionary closure: the -element type is resolved once per column by a 'typeRep' switch and the inner -loop is a monomorphic primop on a 'VU.Vector'. - -Result columns are length @nGroups@ in canonical group order, so they line up -with the key columns 'aggregate' gathers with @selectIndices@. - -The kernel is strictly a FAST PATH: the matcher 'DataFrame.Internal.AggPlan.planAgg' -recognises a small set of expression shapes; anything it does not recognise -keeps the existing interpreter, so the general @aggregate@ API stays correct for -arbitrary expressions. --} +-- | Vectorized scatter-accumulate aggregation kernel. module DataFrame.Internal.AggKernel ( Reduction (..), scatterReduce, @@ -60,10 +42,6 @@ data Reduction | RTop2Sum deriving (Eq, Show) -------------------------------------------------------------------------------- --- Column extraction -------------------------------------------------------------------------------- - {- | Coerce an unboxed Int or Double column to an unboxed Double vector for the moment/mean/sd/median family. Returns 'Nothing' for boxed, nullable, or other element types (the caller then falls back to the interpreter). @@ -79,13 +57,6 @@ scatterColumnToDouble = \case p@(PackedText _ _) -> scatterColumnToDouble (materializePacked p) _ -> Nothing -------------------------------------------------------------------------------- --- Scatter reductions -------------------------------------------------------------------------------- - -{- | Run one fast-path reduction. Returns 'Nothing' when the value column is -not a non-null unboxed Int/Double column (then the caller falls back). --} scatterReduce :: Reduction -> VU.Vector Int -> Int -> Column -> Maybe Column scatterReduce red g nGroups col = case col of @@ -108,10 +79,6 @@ intIdent = Idents maxBound minBound dblIdent :: Idents Double dblIdent = Idents (1 / 0) (negate (1 / 0)) -{- | The monomorphic reduction body. @count@ always yields an Int column; -@sum@/@min@/@max@ preserve the element type; @mean@/@std@/@var@ produce Double; -@top2Sum@ produces Double. --} reduceTyped :: forall a. (Columnable a, VU.Unbox a, Num a, Ord a, Real a) => @@ -183,9 +150,6 @@ meanScatter g nGroups v = runST $ do finalizeMean nGroups s cnt {-# INLINE meanScatter #-} -{- | One pass filling running sum and count from value column @v@ over groups -@g@ into the supplied accumulator arrays. --} scatterSumCount :: (VU.Unbox a, Real a) => VU.Vector Int -> @@ -222,14 +186,6 @@ finalizeMean nGroups s cnt = do go 0 VU.unsafeFreeze out -{- | Sample variance (or its square root for sd) via a per-group Welford -recurrence, scattered into three unboxed arrays @(count, mean, m2)@. This is the -same numerically-stable update as the interpreter's @varianceStep@, so the -result is byte-identical to the existing CollectAgg path (and the db-benchmark -checksum is unchanged); the win is the unboxed scatter replacing the per-group -boxed @VarAcc@ fold over a materialized slice. Degenerate groups (n < 2) yield -0, matching @computeVariance@. --} varScatter :: (VU.Unbox a, Real a) => Bool -> VU.Vector Int -> Int -> VU.Vector a -> VU.Vector Double @@ -268,10 +224,6 @@ varScatter takeSqrt g nGroups v = runST $ do VU.unsafeFreeze out {-# INLINE varScatter #-} -{- | Per-group sum of the two largest values: a 2-slot scatter holding the -running first and second maximum, then @m1 + m2@. Matches the @take 2 . sortBy -(flip compare)@ definition used by the benchmark's @top2Sum@. --} top2Scatter :: (VU.Unbox a, Real a) => VU.Vector Int -> Int -> VU.Vector a -> VU.Vector Double top2Scatter g nGroups v = runST $ do diff --git a/dataframe-core/src/DataFrame/Internal/AggKernelDirect.hs b/dataframe-core/src-internal/DataFrame/Internal/AggKernelDirect.hs similarity index 82% rename from dataframe-core/src/DataFrame/Internal/AggKernelDirect.hs rename to dataframe-core/src-internal/DataFrame/Internal/AggKernelDirect.hs index 67afdb5f..08148949 100644 --- a/dataframe-core/src/DataFrame/Internal/AggKernelDirect.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/AggKernelDirect.hs @@ -5,32 +5,6 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Low-cardinality DIRECT-INDEXED accumulator fast path (research #4). - -When the group-by key resolves to a small dense domain (@nGroups@ below -'directThreshold'), the dense @rowToGroup@ vector handed down by the grouping -layer already IS the group id per row, so we can bypass the @valueIndices@ -gather entirely: scan @rowToGroup@ and the value column /in lockstep, in -original row order/, scattering each value straight into a per-group accumulator -array indexed by the dense id. Both arrays are read sequentially (no random -gather through @valueIndices@), and for a small domain the accumulator stays -cache-resident. - -For parallelism we use the two-phase morsel pattern (#2): split the ROW range -into one contiguous chunk per capability, give each worker a private tiny -accumulator array, scan its chunk sequentially, then merge the @caps@ partials in -a cheap O(caps * nGroups) pass. This makes few-group questions scale (the work -per worker is a tight sequential pass) instead of being dominated by parallel -fan-out, which is exactly the regression the group-range gather suffered on Q4. - -CRITICAL — byte-identical at any @-N@: the two-phase merge only changes the -fold/combine order, so it is admitted ONLY for reductions whose result is -independent of that order: integer sum (exact, associative), count, min, max, -and integer mean (an exact integer sum and count divided once at finalize). The -order-sensitive float reductions (Double sum/mean, variance, sd, top2) are NOT -routed here — the caller keeps the order-preserving group-range kernel for them, -so the db-benchmark checksums stay byte-identical between -N1 and -N8. --} module DataFrame.Internal.AggKernelDirect ( directThreshold, directReduce, @@ -53,10 +27,8 @@ import DataFrame.Internal.Column ( ) {- | Group-domain size at or below which the direct-indexed accumulator path is -taken. The db-benchmark low-cardinality questions (id1=100, id4=100, id6=1e5, -id2:id4=1e4) sit at or below it; wider domains keep the group-range kernel. The -admitted reductions are order-independent, so the per-worker accumulator merge is -exact regardless of size. +taken; wider domains keep the group-range kernel. The admitted reductions are +order-independent, so the per-worker accumulator merge is exact. -} directThreshold :: Int directThreshold = 262144 @@ -72,14 +44,9 @@ the grouping/scatter parallel threshold. parThreshold :: Int parThreshold = 200000 -{- | Run a recognised reduction through the direct-indexed path. Returns -'Nothing' (so the caller falls back to the order-preserving kernel) unless BOTH: -(a) the reduction's result is order-independent at this element type — see the -module note — and (b) the value column is a clean unboxed Int/Double column. - -@g@ is the dense @rowToGroup@ vector (group id per row, original row order); -@nGroups@ is the dense domain size, already verified @<= directThreshold@ by the -caller. +{- | Run a recognised reduction through the direct-indexed path. 'Nothing' (so +the caller falls back to the order-preserving kernel) unless the reduction is +order-independent at this element type AND the column is a clean unboxed Int/Double. -} directReduce :: Reduction -> VU.Vector Int -> Int -> Column -> Maybe Column directReduce red g nGroups col = case col of @@ -116,11 +83,9 @@ directDouble red g nGroups v = case red of shouldPar :: Int -> Bool shouldPar n = n >= parThreshold && capabilities > 1 -{- | Fork @caps@ workers over disjoint contiguous ROW ranges @[lo, hi)@ of -@[0, n)@, balanced evenly by row count; each worker @w@ runs @fill lo hi@ -producing its OWN private accumulator (thread-local, no shared array, no sync). -Returns the partials in worker order for the caller's merge. Rethrows the first -worker failure. +{- | Fork @caps@ workers over disjoint contiguous row ranges of @[0, n)@, each +producing its own private accumulator (no shared array, no sync). Returns the +partials in worker order for the caller's merge; rethrows the first failure. -} runPartialsOver :: Int -> Int -> (Int -> Int -> IO (VUM.IOVector Int)) -> IO [VUM.IOVector Int] diff --git a/dataframe-core/src/DataFrame/Internal/AggKernelPar.hs b/dataframe-core/src-internal/DataFrame/Internal/AggKernelPar.hs similarity index 78% rename from dataframe-core/src/DataFrame/Internal/AggKernelPar.hs rename to dataframe-core/src-internal/DataFrame/Internal/AggKernelPar.hs index 8e8bafd7..db8ae7a7 100644 --- a/dataframe-core/src/DataFrame/Internal/AggKernelPar.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/AggKernelPar.hs @@ -5,27 +5,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Parallel scatter-accumulate aggregation kernel. - -This builds on the sequential kernel ('DataFrame.Internal.AggKernel') and the -Round-5 grouping layout: 'groupBy' hands us @valueIndices@ (rows ordered by -group) and @offsets@ (per-group boundaries), so each group's rows are a -contiguous run @valueIndices[offsets[g] .. offsets[g+1])@ in their original row -order. - -The parallel driver splits the dense group-id range @[0, nGroups)@ into one -contiguous chunk per capability, balanced by row count. Because group ranges are -disjoint and their @valueIndices@ runs are disjoint, every worker reads and -writes its own slice of the shared output array(s) — there is NO cross-worker -overlap and NO merge. And because each group's rows are visited in the same -original-row order as the sequential @rowToGroup@ scan, the per-group fold order -is unchanged, so the result is /byte-identical/ to the sequential kernel at any -@-N@ (no fold-order drift, even for the float sums). - -Forks plain 'forkIO' workers (no sparks), one per capability; falls back to the -sequential 'DataFrame.Internal.AggKernel' path when @caps == 1@ or the row count -is below 'parThreshold'. --} +-- | Parallel scatter-accumulate aggregation kernel. module DataFrame.Internal.AggKernelPar ( scatterReducePar, momentScatterPar, @@ -54,13 +34,6 @@ import DataFrame.Internal.Column ( materializePacked, ) -------------------------------------------------------------------------------- --- Parallelisation policy -------------------------------------------------------------------------------- - -{- | Below this many rows the fork overhead is not worth it; the sequential -kernel runs instead. Mirrors 'DataFrame.Internal.GroupingPar.parThreshold'. --} parThreshold :: Int parThreshold = 200000 @@ -72,21 +45,11 @@ capabilities = unsafePerformIO getNumCapabilities shouldPar :: Int -> Bool shouldPar n = n >= parThreshold && capabilities > 1 -------------------------------------------------------------------------------- --- Group-range partitioning by row count -------------------------------------------------------------------------------- - -{- | Split @[0, nGroups)@ into @caps@ contiguous group ranges, each holding a -near-equal share of rows. Returns @caps + 1@ boundaries @b@ with @b[0] == 0@ and -@b[caps] == nGroups@; worker @w@ owns groups @[b[w], b[w+1])@. A row-balanced -split keeps skew low even when group sizes vary wildly. --} groupRangeBounds :: VU.Vector Int -> Int -> Int -> VU.Vector Int groupRangeBounds offs nGroups caps = VU.create $ do b <- VUM.new (caps + 1) let !nRows = VU.unsafeIndex offs nGroups !per = max 1 ((nRows + caps - 1) `div` caps) - -- First group whose start offset reaches @target@ (>= prev), or nGroups. adv !target !gg | gg >= nGroups = nGroups | VU.unsafeIndex offs gg >= target = gg @@ -102,10 +65,6 @@ groupRangeBounds offs nGroups caps = VU.create $ do go 1 0 pure b -{- | Fork @caps@ workers, worker @w@ running @act (b[w]) (b[w+1])@ over its -disjoint group range; join and rethrow the first failure. Sequential when -@caps == 1@. --} forEachRange :: VU.Vector Int -> Int -> (Int -> Int -> IO ()) -> IO () forEachRange bounds caps act | caps <= 1 = act (VU.unsafeIndex bounds 0) (VU.unsafeIndex bounds caps) @@ -121,15 +80,6 @@ forEachRange bounds caps act _ <- forkIO (try (act s e) >>= putMVar var) pure var -------------------------------------------------------------------------------- --- Parallel single-column reductions -------------------------------------------------------------------------------- - -{- | Parallel counterpart of 'scatterReduce'. Returns 'Nothing' on the same -columns the sequential kernel rejects (boxed/nullable/non-Int-Double); on the -sequential path or tiny inputs it delegates to 'scatterReduce'. The result is -byte-identical to 'scatterReduce' (same per-group fold order). --} scatterReducePar :: Reduction -> VU.Vector Int -> VU.Vector Int -> Int -> Column -> Maybe Column scatterReducePar red vis offs nGroups col @@ -146,9 +96,6 @@ scatterReducePar red vis offs nGroups col _ -> Nothing {-# NOINLINE scatterReducePar #-} -{- | Reconstruct @rowToGroup@ from the group layout for the sequential delegate. -Only used on the small/-N1 path, so the extra pass is negligible. --} rtgFromVis :: VU.Vector Int -> VU.Vector Int -> Int -> VU.Vector Int rtgFromVis vis offs nGroups = VU.create $ do let n = VU.length vis @@ -175,10 +122,6 @@ intIdent = Idents maxBound minBound dblIdent :: Idents Double dblIdent = Idents (1 / 0) (negate (1 / 0)) -{- | The monomorphic parallel reduction body. Each scatter allocates its full -@nGroups@ output array(s) once, then workers fill disjoint group ranges in -parallel. The result type follows the sequential kernel exactly. --} reduceParTyped :: forall a. (Columnable a, VU.Unbox a, Num a, Ord a, Real a) => @@ -314,11 +257,6 @@ meanPar vis offs nGroups v caps bounds = do VU.unsafeFreeze out {-# INLINE meanPar #-} -{- | Per-group Welford variance/sd, parallel by group range. The recurrence is -applied in original-row order within each group, identical to the sequential -@varScatter@, so the result is byte-identical (no parallel-combine needed: each -group lives wholly inside one worker's range). --} varPar :: (VU.Unbox a, Real a) => Bool -> @@ -401,9 +339,8 @@ top2Par vis offs nGroups v caps bounds = do ------------------------------------------------------------------------------- {- | Parallel counterpart of 'momentScatter': one fused pass over both columns, -each group's six sums accumulated in original-row order inside one worker's -range. Byte-identical to 'momentScatter'; delegates to it on the sequential -path. Returns 'Nothing' unless both columns are non-null unboxed Int/Double. +each group's six sums accumulated within one worker's range. Byte-identical to +'momentScatter'. 'Nothing' unless both columns are non-null unboxed Int/Double. -} momentScatterPar :: VU.Vector Int -> VU.Vector Int -> Int -> Column -> Column -> Maybe Moments diff --git a/dataframe-core/src/DataFrame/Internal/AggPlan.hs b/dataframe-core/src-internal/DataFrame/Internal/AggPlan.hs similarity index 83% rename from dataframe-core/src/DataFrame/Internal/AggPlan.hs rename to dataframe-core/src-internal/DataFrame/Internal/AggPlan.hs index 233c2aed..8ad9afff 100644 --- a/dataframe-core/src/DataFrame/Internal/AggPlan.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/AggPlan.hs @@ -8,19 +8,8 @@ {-# LANGUAGE TypeApplications #-} {- | The aggregation fast-path planner and the two-column moment scatter. - -'planAgg' inspects a named output expression and, on a recognised shape over a -clean (non-null, unboxed Int/Double) column, returns the 'AggPlan' the caller -runs through the scatter kernel ('DataFrame.Internal.AggKernel'); anything it -does not recognise returns 'Nothing' so the caller keeps the existing -interpreter. Recognition is by the 'AggStrategy' name tag plus the shape of the -inner 'Expr' — no new constructor is needed, and the general @aggregate@ API is -unchanged. - -'momentScatter' fuses the additive moment sums of two columns (count, Sx, Sy, -Sxx, Syy, Sxy) into one pass — the sufficient statistics for the Q9 regression -family. It is exposed for callers that want to collapse the six separate folds -into a single pass. +'planAgg' recognises a supported aggregate shape over a clean unboxed Int/Double +column and returns an 'AggPlan'; 'momentScatter' fuses the six regression sums. -} module DataFrame.Internal.AggPlan ( AggPlan (..), @@ -65,10 +54,9 @@ data AggPlan | -- | Holistic median over one named column. PlanMedian T.Text -{- | Inspect a named output expression. On a recognised shape over a present -clean column return @Just plan@; otherwise 'Nothing'. Nullable (bitmap) or -non-Int/Double value columns are rejected here so the scatter only ever sees a -clean unboxed vector. +{- | Inspect a named output expression; return @Just plan@ on a recognised shape +over a present clean column, else 'Nothing'. Nullable or non-Int/Double columns +are rejected here so the scatter only sees a clean unboxed vector. -} planAgg :: GroupedDataFrame -> UExpr -> Maybe AggPlan planAgg gdf (UExpr (expr :: Expr a)) = case expr of @@ -120,10 +108,9 @@ isUnboxedNumeric = \case Nothing -> False _ -> False -{- | A recognised moment (Q9 regression) aggregate group: six output columns -that together form the sufficient statistics of two base columns @x@ and @y@. -The caller runs the fused 'momentScatter'/'momentScatterPar' once over -@(colX, colY)@ and binds each output name to the named field of the result. +{- | A recognised moment (Q9 regression) aggregate group: six output columns that +form the sufficient statistics of two base columns @x@ and @y@. The caller runs +'momentScatter' once and binds each output name to a field of the result. -} data MomentPlan = MomentPlan { mpColX :: T.Text @@ -145,12 +132,9 @@ data Term | Prod T.Text T.Text deriving (Eq, Ord, Show) -{- | Recognise the multi-aggregate moment shape across a whole @aggregate@ list: -exactly @count(_)@, @sum(x)@, @sum(y)@, @sum(x*x)@, @sum(y*y)@, @sum(x*y)@ over -two distinct base columns @x@ and @y@ (after resolving derived product columns -through @derivingExpressions@). Returns 'Nothing' on any other set so the caller -falls back to the per-expression planner. Both base columns must be clean -unboxed Int/Double, the same gate the single-column scatter uses. +{- | Recognise the moment shape across a whole @aggregate@ list: exactly +@count@, @sum(x)@, @sum(y)@, @sum(x*x)@, @sum(y*y)@, @sum(x*y)@ over two distinct +clean unboxed base columns. 'Nothing' on any other set. -} planMoments :: GroupedDataFrame -> [(T.Text, UExpr)] -> Maybe MomentPlan planMoments gdf aggs @@ -252,10 +236,9 @@ data Moments = Moments , mSxy :: Column } -{- | One pass over two Double-coercible columns @x@ and @y@ filling the count -and the five sums. Collapses the Q9 regression family's six independent folds -(and three derive passes) into a single fused pass. Returns 'Nothing' unless -both columns are non-null unboxed Int/Double. +{- | One pass over two Double-coercible columns @x@ and @y@ filling the count and +five sums, collapsing the Q9 regression family's six folds into a single pass. +'Nothing' unless both columns are non-null unboxed Int/Double. -} momentScatter :: VU.Vector Int -> Int -> Column -> Column -> Maybe Moments momentScatter g nGroups colX colY = do diff --git a/dataframe-core/src/DataFrame/Internal/Column.hs b/dataframe-core/src-internal/DataFrame/Internal/Column.hs similarity index 90% rename from dataframe-core/src/DataFrame/Internal/Column.hs rename to dataframe-core/src-internal/DataFrame/Internal/Column.hs index a9d83fbd..f374868e 100644 --- a/dataframe-core/src/DataFrame/Internal/Column.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Column.hs @@ -61,21 +61,17 @@ import Type.Reflection -- | A bit-packed validity bitmap. Bit @i@ = 1 means row @i@ is valid (not null). type Bitmap = VU.Vector Word8 -{- | Our representation of a column is a GADT that can store data based on the underlying data. - -This allows us to pattern match on data kinds and limit some operations to only some -kinds of vectors. Nullability is represented via an optional bit-packed 'Bitmap': -@Nothing@ = no nulls; @Just bm@ = bit @i@ of @bm@ is 1 iff row @i@ is valid. +{- | Type-erased column GADT. Pattern-matching on the constructor recovers the +representation; nullability is an optional bit-packed 'Bitmap' (@Nothing@ = no +nulls, @Just bm@ = bit @i@ set iff row @i@ is valid). -} data Column where BoxedColumn :: (Columnable a) => Maybe Bitmap -> VB.Vector a -> Column UnboxedColumn :: (Columnable a, VU.Unbox a) => Maybe Bitmap -> VU.Vector a -> Column - -- Bit-packed Text: a shared UTF-8 byte buffer + (n+1) row offsets + an - -- optional validity bitmap. No per-row Text header; Text is produced only - -- on demand. Behaves as a column of Text (or Maybe Text when a bitmap is - -- present). Only the CSV ingest path emits this; user-built Text columns - -- stay 'BoxedColumn'. + -- Bit-packed Text: shared UTF-8 byte buffer + row offsets + optional bitmap; + -- Text is materialized on demand. Only CSV ingest emits this; user-built + -- Text columns stay 'BoxedColumn'. PackedText :: Maybe Bitmap -> {-# UNPACK #-} !PackedTextData -> Column {- | A mutable companion struct to dataframe columns. @@ -144,12 +140,10 @@ buildBitmapFromNulls n nullIdxs = bitmapSlice :: Int -> Int -> Bitmap -> Bitmap bitmapSlice start len bm | start .&. 7 == 0 = - -- byte-aligned: simple slice; clamp so we never ask for more bytes than exist let startByte = start `shiftR` 3 bytes = min ((len + 7) `shiftR` 3) (VU.length bm - startByte) in VU.slice startByte bytes bm | otherwise = - -- non-aligned: unpack bit-by-bit and repack let n = min len (VU.length bm `shiftL` 3 - start) in buildBitmapFromValid $ VU.generate n $ @@ -168,11 +162,9 @@ bitmapConcat n1 bm1 n2 bm2 = mergeBitmaps :: Bitmap -> Bitmap -> Bitmap mergeBitmaps = VU.zipWith (.&.) -{- | Materialize a nullable column from @VB.Vector (Maybe a)@. -When @a@ is unboxable, creates an 'UnboxedColumn' (more compact). -Otherwise creates a 'BoxedColumn'. -Always attaches a bitmap so the column is recognized as nullable even when -no 'Nothing' values are present (preserves the Maybe type marker). +{- | Materialize a nullable column from @VB.Vector (Maybe a)@; picks 'UnboxedColumn' +when @a@ is unboxable, else 'BoxedColumn'. Always attaches a bitmap so the column +reads as nullable even with no 'Nothing' values. -} fromMaybeVec :: forall a. (Columnable a) => VB.Vector (Maybe a) -> Column fromMaybeVec v = case sUnbox @a of @@ -200,7 +192,7 @@ fromMaybeVecUnboxed v = VU.unsafeFreeze mv in UnboxedColumn (Just bm) dat --- | Materialize an element from a column at index @i@, respecting the bitmap. +-- | Whether row @i@ is null, respecting the bitmap. columnElemIsNull :: Column -> Int -> Bool columnElemIsNull (BoxedColumn (Just bm) _) i = not (bitmapTestBit bm i) columnElemIsNull (UnboxedColumn (Just bm) _) i = not (bitmapTestBit bm i) @@ -213,15 +205,12 @@ columnBitmap (BoxedColumn bm _) = bm columnBitmap (UnboxedColumn bm _) = bm columnBitmap (PackedText bm _) = bm -{- | The universal cold fallback: decode a 'PackedText' into a -@BoxedColumn Text@ using the exact 'sliceTextVector' path the boxed-Text -builder used, so the result is bit-identical to materializing at freeze. -Identity on every other column. +{- | Decode a 'PackedText' into a @BoxedColumn Text@ (bit-identical to +materializing at freeze). Identity on every other column. -} materializePacked :: Column -> Column materializePacked (PackedText bm p) = case packedRowOffsetVec p of Just (arr, offs) -> BoxedColumn bm (sliceTextVector arr offs) - -- Gathered/selected payload: rows are non-contiguous, decode per row. Nothing -> BoxedColumn bm (VB.generate (packedLength p) (packedIndexText p)) materializePacked c = c {-# INLINE materializePacked #-} @@ -236,11 +225,9 @@ isPackedText _ = False -- End bitmap helpers -- --------------------------------------------------------------------------- -{- | A TypedColumn is a wrapper around our type-erased column. -It is used to type check expressions on columns. - -Note: there is no guarantee that the Phanton type is the -same as the underlying vector type. +{- | A wrapper around the type-erased 'Column' carrying a phantom element type, +used to type-check expressions. The phantom is not guaranteed to match the +underlying vector's type. -} data TypedColumn a where TColumn :: (Columnable a) => Column -> TypedColumn a @@ -281,9 +268,8 @@ isNumeric (BoxedColumn _ (_vec :: VB.Vector a)) = case testEquality (typeRep @a) Just Refl -> True isNumeric (PackedText _ _) = False -{- | Checks if a column is of a given type values. -For nullable columns (@BoxedColumn (Just _)@ or @UnboxedColumn (Just _)@), -also returns @True@ when @a = Maybe b@ and the column stores @b@ internally. +{- | Whether the column stores element type @a@. For nullable columns, also +'True' when @a = Maybe b@ and the column stores @b@ internally. -} hasElemType :: forall a. (Columnable a) => Column -> Bool hasElemType = \case @@ -291,10 +277,8 @@ hasElemType = \case UnboxedColumn bm (_column :: VU.Vector b) -> checkUnboxed bm (typeRep @b) PackedText bm _ -> checkBoxed bm (typeRep @T.Text) where - -- Direct type match directMatch :: forall (b :: Type). TypeRep b -> Bool directMatch = isJust . testEquality (typeRep @a) - -- For a nullable column (has bitmap), also accept a = Maybe b checkMaybe :: forall (b :: Type). TypeRep b -> Bool checkMaybe tb = case typeRep @a of App tMaybe tInner -> case eqTypeRep tMaybe (typeRep @Maybe) of @@ -577,12 +561,9 @@ mapColumn f = \case (Columnable a) => Maybe Bitmap -> VB.Vector a -> Either DataFrameException Column runBoxed bm col = case testEquality (typeRep @b) (typeRep @(Maybe a)) of - -- user maps over Maybe a (nullable column as Maybe) Just Refl -> let !n = VB.length col - in -- Build result directly without intermediate Maybe vector to avoid - -- fusion forcing null slots via VU.convert. - Right $ case sUnbox @c of + in Right $ case sUnbox @c of STrue -> UnboxedColumn Nothing $ VU.generate n $ \i -> f @@ -599,7 +580,6 @@ mapColumn f = \case ) Nothing -> case testEquality (typeRep @a) (typeRep @b) of Just Refl -> - -- user maps over inner type a; preserve bitmap Right $ case sUnbox @c of STrue -> UnboxedColumn bm (VU.generate (VB.length col) (f . VB.unsafeIndex col)) SFalse -> case bm of @@ -738,8 +718,6 @@ atIndicesStable indexes (UnboxedColumn bm column) = ) (VU.unsafeBackpermute column indexes) atIndicesStable indexes (PackedText bm p) = - -- Slice-preserving: share the byte buffer, permute via a selection vector - -- instead of materializing boxed Text. Bitmap follows the same gather. PackedText ( fmap ( \bm0 -> @@ -761,9 +739,6 @@ gatherWithSentinel indices col = if VU.unsafeIndex indices i < 0 then 0 else 1 in case col of PackedText srcBm p -> - -- Slice-preserving sentinel gather: share the buffer, permute - -- offsets (negative sentinel -> empty slice), build the null - -- bitmap so -1 rows read as null in left/outer joins. let bm = case srcBm of Nothing -> Just newBm Just sb -> @@ -982,12 +957,9 @@ foldl1DirectGroups f col valueIndices offsets } {-# INLINEABLE foldl1DirectGroups #-} -{- | O(n) fold over groups by scanning the column LINEARLY. -rowToGroup[i] = group index for row i. -Avoids random column reads; random writes go to the accumulator array which is -small (nGroups entries) and typically cache-resident. -When @acc@ is unboxable, uses an unboxed mutable vector for the accumulator -array, eliminating pointer indirection on every read/write. +{- | O(n) fold over groups by scanning the column linearly (rowToGroup[i] = group +of row i). Random writes hit the small per-group accumulator array; when @acc@ is +unboxable that array is unboxed, avoiding pointer indirection. -} foldLinearGroups :: forall b acc. @@ -995,8 +967,8 @@ foldLinearGroups :: (acc -> b -> acc) -> acc -> Column -> - VU.Vector Int -> -- rowToGroup (length n) - Int -> -- nGroups + VU.Vector Int -> + Int -> Either DataFrameException Column foldLinearGroups f seed col rowToGroup nGroups | nGroups == 0 = Right (fromVector @acc VB.empty) @@ -1032,10 +1004,6 @@ foldLinearGroups f seed col rowToGroup nGroups , errorColumnName = Nothing } - -- \| Allocate accumulators, run the traversal, return a frozen Column. - -- When @acc@ is unboxable, uses an unboxed mutable vector (no pointer - -- indirection per read/write) and returns UnboxedColumn directly — - -- avoiding a round-trip through VB.Vector. runWith :: ((Int -> IO acc) -> (Int -> acc -> IO ()) -> IO ()) -> IO Column runWith body = case sUnbox @acc of STrue -> do @@ -1149,13 +1117,11 @@ zipWithColumns :: zipWithColumns f (UnboxedColumn bmL (column :: VU.Vector d)) (UnboxedColumn bmR (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of Just Refl -> case testEquality (typeRep @b) (typeRep @e) of Just Refl - -- Fast path: both plain unboxed, no bitmaps involved in the output type | isNothing bmL , isNothing bmR -> pure $ case sUnbox @c of STrue -> UnboxedColumn Nothing (VU.zipWith f column other) SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other) - -- Type mismatch or bitmap involvement: fall through to general toVector path _ -> zipWithColumnsGeneral f (UnboxedColumn bmL column) (UnboxedColumn bmR other) Nothing -> zipWithColumnsGeneral f (UnboxedColumn bmL column) (UnboxedColumn bmR other) -- TODO: mchavinda - reuse pattern from interpret where we augment the @@ -1239,7 +1205,6 @@ expandColumn n column@(BoxedColumn bm col) Just b -> Just (bitmapConcat (VG.length col) b extra (VU.replicate ((extra + 7) `shiftR` 3) 0)) - -- pad data with default (undefined slot, protected by bitmap) newCol = col <> VB.replicate extra (errorWithoutStackTrace "expandColumn: null slot") in BoxedColumn newBm newCol expandColumn n column@(UnboxedColumn bm col) @@ -1340,13 +1305,9 @@ concatColumns left right = case (left, right) of } ) -{- | Concatenates two columns. - -Works similar to 'concatColumns', but unlike that function, it will also combine columns of different types -by wrapping the values in an Either. - -E.g. combining Column containing [1,2] with Column containing ["a","b"] -will result in a Column containing [Left 1, Left 2, Right "a", Right "b"]. +{- | Like 'concatColumns' but also combines columns of different types by wrapping +values in 'Either' (e.g. @[1,2]@ and @["a","b"]@ become +@[Left 1, Left 2, Right "a", Right "b"]@). -} {- | O(n) Concatenate a list of same-type columns in a single allocation. @@ -1506,34 +1467,14 @@ toList xs = case toVector @a xs of Left err -> throw err Right val -> VB.toList val -{- | Converts a column to a vector of a specific type. - -This is a type-safe conversion that requires the column's element type -to exactly match the requested type. You must specify the desired type -via type applications. - -==== __Type Parameters__ - -[@a@] The element type to convert to -[@v@] The vector type (e.g., 'VU.Vector', 'VB.Vector') - -==== __Examples__ +{- | Type-safe conversion of a column to a vector of element type @a@ (specify via +type application); 'Left' 'TypeMismatchException' when the column's type differs. >>> toVector @Int @VU.Vector column Right (unboxed vector of Ints) >>> toVector @Text @VB.Vector column Right (boxed vector of Text) - -==== __Returns__ - -* 'Right' - The converted vector if types match -* 'Left' 'TypeMismatchException' - If the column's type doesn't match the requested type - -==== __See also__ - -For numeric conversions with automatic type coercion, see 'toDoubleVector', -'toFloatVector', and 'toIntVector'. -} toVector :: forall a v. @@ -1589,23 +1530,9 @@ toVector col = case col of -- Some common types we will use for numerical computing. -{- | Converts a column to an unboxed vector of 'Double' values. - -This function performs intelligent type coercion for numeric types: - -* If the column is already 'Double', returns it directly -* If the column contains other floating-point types, converts via 'realToFrac' -* If the column contains integral types, converts via 'fromIntegral' (beware of overflow if the type is `Integer`). - -==== __Optional column handling__ - -For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number). -This allows optional numeric data to be represented in the resulting vector. - -==== __Returns__ - -* 'Right' - The converted 'Double' vector -* 'Left' 'TypeMismatchException' - If the column is not numeric +{- | Convert a column to an unboxed 'Double' vector, coercing numeric types +('realToFrac' for floats, 'fromIntegral' for integrals; nulls become @NaN@). +'Left' 'TypeMismatchException' when the column is not numeric. -} toDoubleVector :: Column -> Either DataFrameException (VU.Vector Double) toDoubleVector column = @@ -1667,28 +1594,9 @@ toDoubleVector column = } ) -{- | Converts a column to an unboxed vector of 'Float' values. - -This function performs intelligent type coercion for numeric types: - -* If the column is already 'Float', returns it directly -* If the column contains other floating-point types, converts via 'realToFrac' -* If the column contains integral types, converts via 'fromIntegral' -* If the column is boxed 'Integer', converts via 'fromIntegral' (beware of overflow for 64-bit integers and `Integer`) - -==== __Optional column handling__ - -For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number). -This allows optional numeric data to be represented in the resulting vector. - -==== __Returns__ - -* 'Right' - The converted 'Float' vector -* 'Left' 'TypeMismatchException' - If the column is not numeric - -==== __Precision warning__ - -Converting from 'Double' to 'Float' may result in loss of precision. +{- | Convert a column to an unboxed 'Float' vector, coercing numeric types (nulls +become @NaN@); 'Left' 'TypeMismatchException' when not numeric. Converting from +'Double' may lose precision. -} toFloatVector :: Column -> Either DataFrameException (VU.Vector Float) toFloatVector column = @@ -1750,29 +1658,9 @@ toFloatVector column = } ) -{- | Converts a column to an unboxed vector of 'Int' values. - -This function performs intelligent type coercion for numeric types: - -* If the column is already 'Int', returns it directly -* If the column contains floating-point types, rounds via 'round' and converts -* If the column contains other integral types, converts via 'fromIntegral' -* If the column is boxed 'Integer', converts via 'fromIntegral' - -==== __Returns__ - -* 'Right' - The converted 'Int' vector -* 'Left' 'TypeMismatchException' - If the column is not numeric - -==== __Note__ - -Unlike 'toDoubleVector' and 'toFloatVector', this function does NOT support -'OptionalColumn'. Optional columns must be handled separately. - -==== __Rounding behavior__ - -Floating-point values are rounded to the nearest integer using 'round'. -For example: 2.5 rounds to 2, 3.5 rounds to 4 (banker's rounding). +{- | Convert a column to an unboxed 'Int' vector, coercing numeric types +(floats are 'round'ed via banker's rounding); 'Left' 'TypeMismatchException' +when the column is not numeric. Does not support nullable columns. -} toIntVector :: Column -> Either DataFrameException (VU.Vector Int) toIntVector column = diff --git a/dataframe-core/src/DataFrame/Internal/ColumnBuilder.hs b/dataframe-core/src-internal/DataFrame/Internal/ColumnBuilder.hs similarity index 100% rename from dataframe-core/src/DataFrame/Internal/ColumnBuilder.hs rename to dataframe-core/src-internal/DataFrame/Internal/ColumnBuilder.hs diff --git a/dataframe-core/src/DataFrame/Internal/ColumnMerge.hs b/dataframe-core/src-internal/DataFrame/Internal/ColumnMerge.hs similarity index 92% rename from dataframe-core/src/DataFrame/Internal/ColumnMerge.hs rename to dataframe-core/src-internal/DataFrame/Internal/ColumnMerge.hs index 21caf76b..b5ebe58b 100644 --- a/dataframe-core/src/DataFrame/Internal/ColumnMerge.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/ColumnMerge.hs @@ -4,9 +4,8 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Concatenation of per-chunk 'Column's (e.g. from parallel CSV chunks), -re-exported through 'DataFrame.Internal.ColumnBuilder'. Text columns can -merge at the byte level via 'TextChunk' \/ 'mergeTextChunks', so no +{- | Concatenation of per-chunk 'Column's (e.g. from parallel CSV chunks). Text +columns merge at the byte level via 'TextChunk' \/ 'mergeTextChunks', so no per-chunk 'Data.Text.Text' values are ever materialized. -} module DataFrame.Internal.ColumnMerge ( @@ -54,19 +53,17 @@ data TextChunk = TextChunk tcRows :: TextChunk -> Int tcRows c = VU.length (tcOffsets c) - 1 -{- | Freeze a builder chunk directly into a packed-text column: NO -'Data.Text.Text' materialization, NO UTF-8 validation pass (deferred to -decode time). Not yet called by any reader (a later ingest stage flips -'mergeTextChunks' to use it). +{- | Freeze a builder chunk directly into a packed-text column: no +'Data.Text.Text' materialization, no UTF-8 validation pass (deferred to decode). +Not yet called by any reader. -} packedFromTextChunk :: TextChunk -> Column packedFromTextChunk (TextChunk arr _used offs bm) = PackedText bm (mkPackedContiguous arr offs) -{- | Merge text chunks into one packed-text 'Column': one byte-array copy -per chunk, one offset rebase, then wrap the merged shared buffer + offsets -as 'PackedText' (no per-row 'Data.Text.Text' header, no eager UTF-8 -validation pass — decode is deferred to the last mile). +{- | Merge text chunks into one packed-text 'Column': one byte-array copy per +chunk, one offset rebase, then wrap the shared buffer + offsets as 'PackedText' +(no per-row header, decode deferred). -} mergeTextChunks :: [TextChunk] -> Column mergeTextChunks [] = error "DataFrame.Internal.ColumnMerge.mergeTextChunks: empty list" diff --git a/dataframe-core/src/DataFrame/Internal/DataFrame.hs b/dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs similarity index 91% rename from dataframe-core/src/DataFrame/Internal/DataFrame.hs rename to dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs index 46d8937a..951237f1 100644 --- a/dataframe-core/src/DataFrame/Internal/DataFrame.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/DataFrame.hs @@ -35,9 +35,7 @@ import Prelude hiding (null) data DataFrame = DataFrame { columns :: V.Vector Column - {- ^ Our main data structure stores a dataframe as - a vector of columns. This improv - -} + -- ^ Column-oriented storage: the frame is a vector of columns. , columnIndices :: M.Map T.Text Int -- ^ Keeps the column names in the order they were inserted in. , dataframeDimensions :: (Int, Int) @@ -103,14 +101,9 @@ instance Show DataFrame where | otherwise = T.empty in T.unpack (body <> footer) -{- | Configures how a 'DataFrame' is rendered as text. A non-positive value on -any field means \"no limit\" on that axis. - -* 'maxRows' — render at most this many rows from the top of the frame. -* 'maxColumns' — when the frame has more columns than this, the middle columns - are collapsed into a single ellipsis column. -* 'maxCellWidth' — text in any individual cell (including headers and type - rows) longer than this is truncated with a trailing ellipsis. +{- | Configures how a 'DataFrame' is rendered as text: 'maxRows' caps rendered +rows, 'maxColumns' collapses middle columns past the limit into an ellipsis, and +'maxCellWidth' truncates long cells. A non-positive field means \"no limit\". -} data TruncateConfig = TruncateConfig { maxRows :: Int @@ -185,8 +178,6 @@ asTextWith fmt mTrunc d = getType (PackedText Nothing _) = T.pack $ show (typeRep @T.Text) getType (PackedText (Just _) _) = T.pack $ showMaybeType @T.Text - -- Separate out cases dynamically so we don't end up making round trip - -- string copies. get :: Maybe Column -> V.Vector T.Text get (Just (BoxedColumn (Just bm) (column :: V.Vector a))) = V.generate (V.length column) $ \i -> @@ -214,10 +205,9 @@ asTextWith fmt mTrunc d = (map clipCell finalTypes) (map (V.map clipCell) finalCols) -{- | Decide which columns survive horizontal truncation and where (if anywhere) -to splice in the ellipsis column. The split puts the extra column on the -left for odd 'maxColumns'; the ellipsis is only inserted when it actually -saves space (i.e. the frame has more than 'maxColumns' + 1 columns). +{- | Decide which columns survive horizontal truncation and where to splice the +ellipsis column. Splits with the extra column on the left for odd 'maxColumns'; +inserts the ellipsis only when it actually saves space. -} pickColumns :: Maybe TruncateConfig -> @@ -311,14 +301,8 @@ getColumn name df i <- columnIndices df M.!? name columns df V.!? i -{- | Retrieves a column by name from the dataframe, throwing an exception if not found. - -This is an unsafe version of 'getColumn' that throws 'ColumnsNotFoundException' -if the column does not exist. Use this when you are certain the column exists. - -==== __Throws__ - -* 'ColumnsNotFoundException' - if the column with the given name does not exist +{- | Retrieve a column by name, throwing 'ColumnsNotFoundException' if it does not +exist. Unsafe version of 'getColumn'; use when the column is certain to exist. -} unsafeGetColumn :: T.Text -> DataFrame -> Column unsafeGetColumn name df = case getColumn name df of diff --git a/dataframe-core/src/DataFrame/Internal/DictEncode.hs b/dataframe-core/src-internal/DataFrame/Internal/DictEncode.hs similarity index 60% rename from dataframe-core/src/DataFrame/Internal/DictEncode.hs rename to dataframe-core/src-internal/DataFrame/Internal/DictEncode.hs index 6e6f6a6d..7896da88 100644 --- a/dataframe-core/src/DataFrame/Internal/DictEncode.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/DictEncode.hs @@ -4,28 +4,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Dictionary-encode a text (or factor) group key to dense @Int@ codes -(research #4 / #5). - -A text group key is hashed and bucketed exactly as the grouping hash table does, -but instead of carrying the full grouping output it only assigns each row a dense -first-appearance code @0 .. card-1@ (a NULL row gets its own reserved code) and -reports the cardinality. The intent was to feed those codes to the int fast path -(direct-indexed low-card, or a packed composite key) so a text group key reaches -it. - -VERDICT (this round): routing through the codes profiled SLOWER than the existing -hash group-by on EVERY db-benchmark group-by question, so -'DataFrame.Internal.Grouping' does not take the dict path (see 'tryDictGroup' -there). The dict-build is its own hash pass over every row, and the hash -group-by already fuses hashing and grouping into one pass; substituting int codes -adds the encode pass without removing the dominant grouping work. Single low-card -(Q1 id1), single high-card (Q3/Q7 id3) and the composites (Q2 id1:id2, Q10 six -keys) were each measured and all lost. - -This module remains a correct, unit-tested building block: it produces the codes -and the cardinality only; the routing decision lives in -'DataFrame.Internal.Grouping'. +{- | Dictionary-encode a text (or factor) group key to dense @Int@ codes: each row +gets a first-appearance code @0..card-1@ (NULL reserved) plus the cardinality. A +tested building block; profiled slower than the hash group-by, so unused for now. -} module DataFrame.Internal.DictEncode ( dictEncodeColumn, @@ -52,31 +33,22 @@ import DataFrame.Internal.PackedText ( ) {- | Largest distinct-value count we will dictionary-encode. Above this the codes -no longer index a reasonable direct accumulator and the dict-build pass is pure -overhead, so the caller keeps the plain hash group-by. Matches the direct-group -histogram budget. +no longer index a reasonable direct accumulator and the encode pass is pure +overhead, so the caller keeps the plain hash group-by. -} dictMaxCardinality :: Int dictMaxCardinality = 1048576 -{- | Dictionary-encode a text-like column to dense first-appearance @Int@ codes. - -Returns @Just (codes, cardinality)@ where @codes!i@ is the dense id of row @i@'s -value (a NULL row, when the column is nullable, is assigned its own reserved code -distinct from every present value) and @cardinality@ is the number of distinct -codes used. Returns 'Nothing' for any non-text column or when the cardinality -exceeds 'dictMaxCardinality' (so the caller falls back to the hash group-by). - -Only 'PackedText' and boxed 'Data.Text.Text' columns are encoded; everything else -is 'Nothing'. +{- | Dictionary-encode a text-like column to dense first-appearance @Int@ codes, +returning @Just (codes, cardinality)@ (a NULL row gets its own reserved code). +'Nothing' for non-text columns or cardinality above 'dictMaxCardinality'. -} dictEncodeColumn :: Column -> Maybe (VU.Vector Int, Int) dictEncodeColumn = dictEncodeColumnUpTo dictMaxCardinality {- | Dictionary-encode like 'dictEncodeColumn' but bail to 'Nothing' as soon as -the distinct count would exceed @maxCard@. The early bail lets a low-cardinality -PROBE (the single-key direct path) avoid a full high-cardinality pass when the -column turns out to be high-card. +the distinct count would exceed @maxCard@, letting a low-cardinality probe avoid +a full high-cardinality pass. -} dictEncodeColumnUpTo :: Int -> Column -> Maybe (VU.Vector Int, Int) dictEncodeColumnUpTo maxCard (PackedText bm p) = encodePacked maxCard bm p @@ -86,10 +58,9 @@ dictEncodeColumnUpTo maxCard (BoxedColumn bm (v :: V.Vector a)) = Nothing -> Nothing dictEncodeColumnUpTo _ _ = Nothing -{- | Encode a packed-text column. Hashes each row's raw UTF-8 bytes (the same -'mixBytes' the grouping hash uses) and re-verifies byte equality on collisions, -assigning dense codes in first-appearance order. A null row hashes 'nullSalt' and -re-verifies as equal-to-null only. +{- | Encode a packed-text column: hash each row's raw UTF-8 bytes (the grouping +'mixBytes'), re-verify byte equality on collisions, assign dense codes in +first-appearance order. A null row hashes 'nullSalt'. -} encodePacked :: Int -> Maybe Bitmap -> PackedTextData -> Maybe (VU.Vector Int, Int) @@ -131,10 +102,9 @@ encodeBoxedText maxCard bm v = _ -> False in buildCodes maxCard n hashAt eqAt -{- | The shared code-assignment loop. Buckets every row through an -open-addressing table on its precomputed hash, re-verifying the real value with -@eqAt@ on a hash hit, assigning dense first-appearance codes. Bails to 'Nothing' -the moment the distinct count would exceed 'dictMaxCardinality'. +{- | The shared code-assignment loop: bucket every row through an open-addressing +table on its precomputed hash, re-verify with @eqAt@ on a hit, assign dense +first-appearance codes. Bails to 'Nothing' once the distinct count exceeds @maxCard@. -} buildCodes :: Int -> Int -> (Int -> Int) -> (Int -> Int -> Bool) -> Maybe (VU.Vector Int, Int) diff --git a/dataframe-core/src/DataFrame/Internal/Expression.hs b/dataframe-core/src-internal/DataFrame/Internal/Expression.hs similarity index 89% rename from dataframe-core/src/DataFrame/Internal/Expression.hs rename to dataframe-core/src-internal/DataFrame/Internal/Expression.hs index ef43f068..0a3c2e23 100644 --- a/dataframe-core/src/DataFrame/Internal/Expression.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Expression.hs @@ -25,11 +25,9 @@ import DataFrame.Internal.Column import qualified DataFrame.Internal.Pretty as P import Type.Reflection (Typeable, typeOf, typeRep) -{- | Operators are an open typeclass: built-ins get their own 'Typeable' type so -the simplifier can match them by 'cast', and users can add @instance@s. The -generic 'UnUDF'/'BinUDF' carriers cover UDFs, dynamic-named, and arithmetic ops. -Method names match the carrier record fields ('NoFieldSelectors' frees them), so -existing construction and read sites are unchanged. +{- | Operators are an open typeclass: built-ins get their own 'Typeable' type so the +simplifier can match them by 'cast', and users can add instances. The generic +'UnUDF'/'BinUDF' carriers cover UDFs, dynamic-named, and arithmetic ops. -} class (Typeable op) => UnaryOp op where unaryFn :: op a b -> a -> b @@ -303,7 +301,7 @@ normalize expr = case expr of Nothing -> expr Just Refl -> if compareExpr n1 n2 == GT - then Binary op n2 n1 -- Swap to canonical order + then Binary op n2 n1 else Binary op n1 n2 | otherwise -> Binary op (normalize e1) (normalize e2) Agg strat e -> Agg strat (normalize e) @@ -373,17 +371,9 @@ replaceExpr new old expr = case testEquality (typeRep @b) (typeRep @c) of (Agg op inner) -> Agg op (replaceExpr new old inner) (Over keys inner) -> Over keys (replaceExpr new old inner) -{- | Simultaneously substitute column references using a map from column name to -replacement expression. Unlike folding 'replaceExpr' over the bindings, this is a -single parallel pass: every 'Col' reference is resolved against the original map, -so a swap such as @{a ↦ col b, b ↦ col a}@ is handled correctly (sequential -replacement would collapse both columns onto one). - -Only 'Col' references are substituted; raw-text references inside 'CastWith' and -'Over' partition keys are left untouched (documented limitation — the fitted ML -transforms that use this only ever emit @Col@/@Lit@/@Unary@/@Binary@/@If@). A -binding whose replacement type does not match the referenced column's type is a -programmer error and raises an exception. +{- | Simultaneously substitute 'Col' references from a name→expression map in a +single parallel pass, so a swap like @{a ↦ col b, b ↦ col a}@ works. Raw-text +references (in 'CastWith', 'Over' keys) are left untouched; type mismatch raises. -} substituteColumns :: forall a. (Columnable a) => M.Map T.Text UExpr -> Expr a -> Expr a @@ -439,17 +429,13 @@ width ('P.defaultWidth'). See 'prettyPrintWidth' to control wrapping. prettyPrint :: Expr a -> String prettyPrint = prettyPrintWidth P.defaultWidth -{- | Render an expression as readable pseudo-code, wrapping long binary chains -onto aligned continuation lines once they exceed @width@ columns. @if@/@then@/ -@else@ always break onto their own lines and nested @else if@ form a flat -ladder (no staircase indentation). Sub-expressions are parenthesized by operator -precedence so grouping is unambiguous. +{- | Render an expression as readable, width-aware pseudo-code: long binary chains +wrap onto aligned continuation lines, @if@/@then@/@else@ break onto their own lines +(nested @else if@ form a flat ladder), and sub-exprs are parenthesized by precedence. -} prettyPrintWidth :: Int -> Expr a -> String prettyPrintWidth width = P.render width . toDoc 0 where - -- \| @toDoc prec e@ renders @e@, parenthesizing when the enclosing operator - -- precedence @prec@ exceeds @e@'s own. toDoc :: Int -> Expr x -> P.Doc toDoc prec expr = case expr of Col name -> P.text (T.unpack name) @@ -471,8 +457,6 @@ prettyPrintWidth width = P.render width . toDoc 0 Over keys inner -> toDoc 0 inner <> P.text (".over(" ++ show (map T.unpack keys) ++ ")") - -- \| Commutative (hence associative here) operators flatten into a single - -- operator-led, wrappable chain; others render as a binary pair. renderBinary :: (BinaryOp op) => Int -> op c b a -> String -> Expr c -> Expr b -> P.Doc renderBinary prec op sym l r = @@ -493,16 +477,10 @@ prettyPrintWidth width = P.render width . toDoc 0 _ -> toDoc p l P.<+> P.text sym P.<+> toDoc p r | otherwise = P.group (toDoc p l <> P.nest 2 (P.line <> P.text sym P.<+> toDoc p r)) - in -- @prec > p@: precedence forces parens even on one line. @prec < p@ - -- (and we are nested in some binary, @prec >= 1@): precedence alone - -- does not require parens, but a wrapped operand is ambiguous, so - -- parenthesize only when it breaks across lines. - if prec > p + in if prec > p then P.parens body else if prec >= 1 && prec < p then P.parensWhenBroken body else body - -- \| Collect operands of a same-name, same-precedence commutative operator - -- chain in left-to-right order. flattenChain :: T.Text -> Int -> Expr x -> [P.Doc] flattenChain name p e = case e of Binary op' l' r' @@ -523,7 +501,6 @@ prettyPrintWidth width = P.render width . toDoc 0 in if prec > 0 then P.parens (P.nest 2 blk) else blk renderIf _ _ = mempty - -- \| Flatten an @else if@ chain so each level stays at the same indent. renderElse :: Expr x -> P.Doc renderElse (If c t e) = P.text "else if" P.<+> P.nest 8 (P.group (toDoc 0 c)) diff --git a/dataframe-core/src/DataFrame/Internal/Grouping.hs b/dataframe-core/src-internal/DataFrame/Internal/Grouping.hs similarity index 75% rename from dataframe-core/src/DataFrame/Internal/Grouping.hs rename to dataframe-core/src-internal/DataFrame/Internal/Grouping.hs index 2114f5dc..5e2bc094 100644 --- a/dataframe-core/src/DataFrame/Internal/Grouping.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Grouping.hs @@ -52,16 +52,9 @@ import DataFrame.Internal.Types import System.IO.Unsafe (unsafePerformIO) import Type.Reflection (typeRep) -{- | O(k * n) groups the dataframe by the given rows aggregating the remaining rows -into vector that should be reduced later. - -Rows are bucketed with an unboxed open-addressing hash table -('DataFrame.Internal.HashTable') that maps each row's key-hash to a dense group -id, re-verifying the real key columns on every hash hit. This both cuts the -per-row boxed allocation of the previous 'Data.IntMap' bucketing (less GC) and -fixes a latent collision bug where two distinct keys sharing a hash were merged. -Groups are numbered in first-appearance order; 'valueIndices' / 'offsets' are -then derived by a stable counting sort on the group id. +{- | O(k * n) group the dataframe by the given key columns, bucketing rows with an +open-addressing hash table that re-verifies keys on each hash hit. Groups are +numbered in first-appearance order; 'valueIndices'/'offsets' follow by counting sort. -} groupBy :: [T.Text] -> @@ -87,11 +80,9 @@ groupBy names df where !n = nRows df -{- | The low-cardinality direct-indexed grouping fast path -('DataFrame.Internal.GroupingDirect'). Fires only for a SINGLE clean small-range -@Int@ key column; one shared function feeds both -N1 and -N8 so the result is -identical at any capability count (parallel==sequential by construction). Returns -'Nothing' on any other key shape, falling through to the hash group-by. +{- | Low-cardinality direct-indexed grouping fast path +('DataFrame.Internal.GroupingDirect'): fires only for a single clean small-range +@Int@ key. Returns 'Nothing' on any other key shape, falling back to the hash path. -} tryDirectGroup :: [T.Text] -> DataFrame -> Maybe GroupedDataFrame tryDirectGroup [name] df = do @@ -102,22 +93,9 @@ tryDirectGroup [name] df = do Nothing -> tryDictGroup (nRows df) df [name] col tryDirectGroup _ _ = Nothing -{- | Dictionary-encode a single text key to dense int codes (the codes ARE the -first-appearance group ids), then derive @valueIndices@/@offsets@ from those -codes with one counting sort. - -PROFILED AS A LOSS, so this currently always falls back ('Nothing'). The -dict-build is its own hash pass over every row, and the hash group-by already -hashes and groups in one fused pass: at -N1 the dict path measured ~0.44s vs -~0.33s for the hash path on Q1 (id1, 100 groups) at 1e7 rows, and at -N8 the -parallel partitioned grouping is far faster than any sequential dict-build. The -high-card single keys (Q3/Q7 id3 ~1e5) and the multi-key composites (Q2 id1:id2, -Q10 six keys) were each tried and also lost — substituting int codes does not -remove the dominant grouping passes, it adds the encode passes on top. The -'DataFrame.Internal.DictEncode.dictEncodeColumnUpTo' step is kept and unit-tested -as a correct building block; the routing here is deliberately disabled. The -@_n@/@_df@/@_names@/@_col@ wiring is retained so re-enabling is a one-line change -should a parallel dict-encode ever change the verdict. +{- | Dictionary-encode a single text key to dense int codes, then derive +@valueIndices@/@offsets@ by counting sort. Profiled slower than the fused hash +group-by on every db-benchmark question, so it always falls back ('dictGroupEnabled'). -} tryDictGroup :: Int -> DataFrame -> [T.Text] -> Column -> Maybe GroupedDataFrame @@ -135,9 +113,8 @@ it profiled slower than the hash group-by on every db-benchmark group-by questio dictGroupEnabled :: Bool dictGroupEnabled = False -{- | Cardinality ceiling the single-key dict-encode probe would use: it bails to -'Nothing' once the distinct count passes this, so a high-card key never pays for -a full encode pass. Only consulted when 'dictGroupEnabled' is 'True'. +{- | Cardinality ceiling for the single-key dict-encode probe: it bails to 'Nothing' +once the distinct count passes this. Only consulted when 'dictGroupEnabled' is 'True'. -} dictSingleThreshold :: Int dictSingleThreshold = 4096 @@ -157,10 +134,9 @@ groupBySeq names df = (vis, os) = indicesFromGroups rtg nGroups in Grouped df names vis os rtg -{- | The parallel partitioned grouping path (see -'DataFrame.Internal.GroupingPar'). Forks one task per capability; produces an -output bit-for-bit identical to 'groupBySeq'. Pure via 'unsafePerformIO' (the IO -is deterministic thread fan-out only). +{- | The parallel partitioned grouping path (see 'DataFrame.Internal.GroupingPar'): +forks one task per capability, producing output bit-for-bit identical to +'groupBySeq'. Pure via 'unsafePerformIO' (deterministic thread fan-out only). -} groupByPar :: [T.Text] -> DataFrame -> GroupedDataFrame groupByPar names df = @@ -177,11 +153,9 @@ keyColIndices :: [T.Text] -> DataFrame -> [Int] keyColIndices names df = M.elems $ M.filterWithKey (\k _ -> k `elem` names) (columnIndices df) -{- | Assign every row to a dense group id via the open-addressing table, in -first-appearance order. Returns @(rowToGroup, repHash, repRow)@ where @repHash@ / -@repRow@ are the hash and representative row index of each group (indexed by the -first-appearance id). The table re-verifies the real key with 'eqKeyRow' on each -hash hit so colliding keys are kept apart. +{- | Assign every row to a dense group id in first-appearance order. Returns +@(rowToGroup, repHash, repRow)@ — the hash and representative row of each group. +'eqKeyRow' re-verifies the real key on each hash hit so colliding keys stay apart. -} assignGroups :: DataFrame -> [Int] -> Int -> (VU.Vector Int, VU.Vector Int, VU.Vector Int) @@ -190,7 +164,6 @@ assignGroups df indicesToGroup n = runST $ do let !eqRow = eqKeyRow df indicesToGroup ht <- newHashTable n rtg <- VUM.new n - -- At most n groups; trimmed to the actual count on freeze. repHashM <- VUM.new n repRowM <- VUM.new n let go !i !next @@ -209,19 +182,9 @@ assignGroups df indicesToGroup n = runST $ do repRow <- VU.unsafeFreeze (VUM.slice 0 nGroups repRowM) pure (frozen, repHash, repRow) -{- | Map each first-appearance group id to its canonical id: groups are ordered -by ascending representative hash, tie-broken by representative row index. This -makes the emitted group order a deterministic function of the key set (not of -input row order), so set operations like @union a b@ and @union b a@ agree, and -reproduces the ascending-hash order of the previous 'Data.IntMap' grouping. -Returns @remap@ with @remap[firstAppearanceId] = canonicalId@. - -The ordering is the stable hash-rank of 'DataFrame.Internal.RadixRank': @repRow@ -is strictly ascending in first-appearance id order (a new group's representative -is the first row that reaches it, scanned in increasing row index), so the stable -sort's equal-hash tie-break reproduces the old @(hash, repRow)@ comparison. O(g) -with no boxed-tuple comparison sort — this keeps the @1e7@-distinct-group case -(Q10) off an @n log n@ list sort. +{- | Map each first-appearance group id to its canonical id: groups ordered by +ascending representative hash (tie-broken by representative row), making group +order a deterministic function of the key set so set ops commute. O(g), no sort. -} canonicalRemap :: VU.Vector Int -> VU.Vector Int -> VU.Vector Int canonicalRemap repHash _repRow = @@ -278,9 +241,8 @@ computeHashes df indicesToGroup n = do VU.unsafeFreeze mh {- | Build the row-key equality predicate over the selected key columns. -@eqKeyRow df idxs a b@ is 'True' iff rows @a@ and @b@ are equal across all key -columns, comparing validity bits first (a null equals only another null) then -the underlying value. Used by the hash table to reject hash collisions. +@eqKeyRow df idxs a b@ is 'True' iff rows @a@ and @b@ agree on all key columns +(validity first, a null equals only a null). Used to reject hash collisions. -} eqKeyRow :: DataFrame -> [Int] -> Int -> Int -> Bool eqKeyRow df indicesToGroup = @@ -326,8 +288,6 @@ single placement pass keeps rows in original order within each group. indicesFromGroups :: VU.Vector Int -> Int -> (VU.Vector Int, VU.Vector Int) indicesFromGroups rtg nGroups = runST $ do let !n = VU.length rtg - -- counts[g] = size of group g (g in [0, nGroups)). Slot nGroups stays 0 so - -- the exclusive scan below lands n in offsets[nGroups]. counts <- VUM.replicate (nGroups + 1) 0 let countLoop !i | i >= n = pure () @@ -337,8 +297,6 @@ indicesFromGroups rtg nGroups = runST $ do VUM.unsafeWrite counts g (c + 1) countLoop (i + 1) countLoop 0 - -- Exclusive prefix scan of counts into offsets: offsets[k] is the start of - -- group k and offsets[nGroups] == n. offsM <- VUM.new (nGroups + 1) let scan !k !acc | k > nGroups = pure () @@ -347,8 +305,6 @@ indicesFromGroups rtg nGroups = runST $ do c <- VUM.unsafeRead counts k scan (k + 1) (acc + c) scan 0 0 - -- 'counts' is repurposed as a per-group write cursor seeded at each group's - -- start offset, giving a stable placement (rows keep original order). let seed !k | k > nGroups = pure () | otherwise = do diff --git a/dataframe-core/src/DataFrame/Internal/GroupingDirect.hs b/dataframe-core/src-internal/DataFrame/Internal/GroupingDirect.hs similarity index 81% rename from dataframe-core/src/DataFrame/Internal/GroupingDirect.hs rename to dataframe-core/src-internal/DataFrame/Internal/GroupingDirect.hs index a54eb0d8..7a882cba 100644 --- a/dataframe-core/src/DataFrame/Internal/GroupingDirect.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/GroupingDirect.hs @@ -4,29 +4,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Low-cardinality DIRECT-INDEXED grouping fast path (research #4). - -When the group-by key is a single clean (non-null) unboxed @Int@ column whose -value /range/ is small (@max - min + 1 <= directGroupThreshold@), the value -itself indexes a dense accumulator: there is no hashing, no key re-verification, -and no open-addressing probe. This is the X100 / ClickHouse FixedHashMap idea -applied to the grouping step — the dominant cost of the low-cardinality -db-benchmark questions (id4=100, id6=1e5) is the hash group-by, not the -aggregate scatter, so bypassing the hash here is the real lever. - -The pipeline is three linear passes plus an O(range) compaction: - - 1. min/max of the key (parallel range-reduce, order-independent). - 2. a per-value histogram (parallel per-thread histograms, exact-integer merge). - 3. compact non-empty values into dense ids in ASCENDING value order, then a - stable placement pass building @valueIndices@. - -The emitted group order is ascending key value — a deterministic function of the -key set (like the hash path's canonical order), so set operations stay -commutative and the db-benchmark checksums (order-independent sums) are -unchanged. Crucially this single function is shared by both the sequential and -parallel 'groupBy' entry points, so the parallel==sequential parity is automatic -(identical output by construction) at any @-N@. +{- | Low-cardinality direct-indexed grouping fast path: when the key is a single +clean unboxed @Int@ column of small value range, the value itself indexes a dense +accumulator (no hashing/probing). Emits groups in ascending value order. -} module DataFrame.Internal.GroupingDirect ( directGroupThreshold, @@ -122,25 +102,19 @@ shouldPar :: Int -> Bool shouldPar n = n >= parThreshold && capabilities > 1 {- | Build the grouping by counting sort on @value - min@: a (parallel) per-value -histogram, compaction of non-empty values into ascending dense ids, an exclusive -scan into offsets, then a stable placement pass building @valueIndices@ and -@rowToGroup@. +histogram, compaction of non-empty values into ascending dense ids, a scan into +offsets, then a stable placement pass building @valueIndices@ and @rowToGroup@. -} directGroup :: VU.Vector Int -> Int -> Int -> DirectGrouping directGroup v mn range = unsafePerformIO $ do let !n = VU.length v - -- 1. Per-value histogram over the dense value index (parallel, exact). hist <- buildHistogram v mn range n - -- 2. Compact non-empty values -> dense group ids (ascending value order), - -- recording each value's group id and the group's row count. valToGroup <- VUM.replicate range (-1 :: Int) grpCount <- VUM.new range nGroups <- compact hist range valToGroup grpCount - -- 3. Exclusive prefix scan of group counts -> offsets (length nGroups + 1). offsM <- VUM.new (nGroups + 1) cursor <- VUM.new nGroups scanOffsets grpCount nGroups offsM cursor - -- 4. Stable placement: rowToGroup[i] and valueIndices in group order. rtg <- VUM.new n vis <- VUM.new n place v mn n valToGroup cursor rtg vis diff --git a/dataframe-core/src/DataFrame/Internal/GroupingPar.hs b/dataframe-core/src-internal/DataFrame/Internal/GroupingPar.hs similarity index 72% rename from dataframe-core/src/DataFrame/Internal/GroupingPar.hs rename to dataframe-core/src-internal/DataFrame/Internal/GroupingPar.hs index 1d77cce6..36cf28ed 100644 --- a/dataframe-core/src/DataFrame/Internal/GroupingPar.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/GroupingPar.hs @@ -2,26 +2,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE Strict #-} -{- | -Parallel partitioned group-by assignment. Row indices are partitioned by the -high bits of their key-hash (a counting sort: per-range histogram, prefix-sum, -scatter into one index buffer laid out partition-by-partition). One task per -capability then groups its partitions with its OWN open-addressing hash table -('DataFrame.Internal.HashTable') — keys are disjoint across partitions, so the -per-partition group sets concatenate with NO merge. - -The output @(rowToGroup, valueIndices, offsets)@ is /bit-for-bit identical/ to -the sequential 'DataFrame.Internal.Grouping.groupBy': groups are emitted in -ascending @(repHash, repRow)@ order (signed-Int order on the hash, tie-broken by -the representative row). Because the partition key is the top bits of a -sign-preserving unsigned remap of the same hash, partition order already agrees -with that global order; within a partition we sort the local groups by the same -key. This is the parallel==sequential correctness gate. - -The driver forks plain 'forkIO' workers (no sparks) over a shared atomic-counter -work queue, so partition skew is balanced. A sequential fallback is used when -there is a single capability or the row count is below 'parThreshold' (decided in -'shouldParallelize', which 'groupBy' consults before calling here). +{- | Parallel partitioned group-by: rows are counting-sorted into partitions by the +top hash bits, then one task per capability groups its partitions independently. +Output is bit-for-bit identical to the sequential 'DataFrame.Internal.Grouping.groupBy'. -} module DataFrame.Internal.GroupingPar ( parallelAssignGroups, @@ -107,11 +90,7 @@ parallelAssignGroups n hashes eqRow = do caps <- getNumCapabilities let !p = numPartitionsFor caps !shift = 64 - intLog2 p - -- Phase 1: counting sort of row indices by partition. (partStart, sortedRows) <- partitionRows n hashes p shift - -- Phase 2: per-partition grouping + per-partition canonical ranking (both - -- inside the parallel worker). localGid[pos] = local group id of the row at - -- sorted position 'pos'; canonBoxes[part] = its rank vector. localGid <- VUM.new (max 1 n) canonBoxes <- VM.replicate p (VU.empty :: VU.Vector Int) nLocalGroups <- VUM.replicate p (0 :: Int) @@ -125,7 +104,6 @@ parallelAssignGroups n hashes eqRow = do localGid canonBoxes nLocalGroups - -- Phase 3: global base ids (serial prefix sum; the ranking is already done). (globalBase, canonOf, nGroups) <- canonicalize p canonBoxes nLocalGroups assemble n p partStart sortedRows localGid globalBase canonOf nGroups @@ -177,17 +155,9 @@ partitionRows n hashes p shift = do -- Phase 2: per-partition grouping (parallel) ------------------------------------------------------------------------------- -{- | Group each partition with its own hash table, then rank its local groups -into canonical order — all inside the parallel worker. Forks @caps@ workers that -pull partition indices off a shared counter. For partition @pp@ spanning -@[partStart[pp], partStart[pp+1])@ of @sortedRows@ a worker assigns dense local -group ids (first-appearance order) into @localGid@ at the same sorted positions, -recording each new group's representative hash and row into unboxed -first-appearance vectors. It then computes @canonBoxes[pp]@ — @canon[localGid] = -within-partition canonical rank — via a stable radix sort on the unsigned -representative hash (ties keep first-appearance order, which is ascending repRow, -so the @(key hash, repRow)@ order of the old comparison sort is reproduced with -no boxed tuples). @nLocalGroups[pp]@ holds the group count. +{- | Group each partition with its own hash table, then rank its local groups into +canonical order — all inside the parallel worker. Forks @caps@ workers pulling +partition indices off a shared counter; disjoint keys mean no cross-partition merge. -} runPartitions :: Int -> @@ -208,10 +178,6 @@ runPartitions caps p partStart sortedRows hashes eqRow localGid canonBoxes nLoca !sz = e - s when (sz > 0) $ do ht <- newHashTable sz - -- repHash indexed by local gid (first-appearance order). The - -- representative row is implicitly ascending in gid order (sorted - -- positions are in original-row order, a stable counting sort), - -- so the stable hash-rank below needs no explicit repRow. repHashM <- VUM.new sz let loop !pos !nextGid | pos >= e = pure nextGid @@ -227,10 +193,6 @@ runPartitions caps p partStart sortedRows hashes eqRow localGid canonBoxes nLoca else loop (pos + 1) nextGid ng <- loop s 0 VUM.unsafeWrite nLocalGroups pp ng - -- Rank this partition's groups into canonical order (shared with - -- the sequential path). repRow is ascending in gid order (stable - -- counting sort keeps sorted positions in original-row order), so - -- the stable hash-rank's tie-break reproduces (hash, repRow). canon <- rankByHash (VUM.unsafeRead repHashM) ng VM.unsafeWrite canonBoxes pp canon worker = do @@ -243,10 +205,8 @@ runPartitions caps p partStart sortedRows hashes eqRow localGid canonBoxes nLoca ------------------------------------------------------------------------------- {- | Exclusive prefix sum of the per-partition group counts into @globalBase@ -(length @p+1@, @globalBase[pp]@ is the first global id of partition @pp@, -@globalBase[p]@ the total). The per-partition canonical ranks were already -computed in 'runPartitions'; partitions are in ascending key order so prepending -@globalBase[pp]@ to each rank yields the sequential @canonicalRemap@ order. +(@globalBase[pp]@ = first global id of partition @pp@). Ranks were computed in +'runPartitions'; prepending the base to each yields the sequential order. -} canonicalize :: Int -> @@ -266,11 +226,9 @@ canonicalize p canonBoxes nLocalGroups = do canonOf <- V.unsafeFreeze canonBoxes pure (globalBase, canonOf, total) -{- | Build the final @(rowToGroup, valueIndices, offsets)@. For each sorted -position we know its partition, its local group id and the canonical maps, so the -global group id is @globalBase[pp] + canonOf[pp][localGid]@. @valueIndices@ is the -rows ordered by global group; @offsets@ the per-group boundaries; @rowToGroup@ the -inverse mapping per original row. +{- | Build the final @(rowToGroup, valueIndices, offsets)@: the global group id of a +sorted position is @globalBase[pp] + canonOf[pp][localGid]@. @valueIndices@ orders +rows by group, @offsets@ the boundaries, @rowToGroup@ the inverse per original row. -} assemble :: Int -> @@ -284,7 +242,6 @@ assemble :: IO (VU.Vector Int, VU.Vector Int, VU.Vector Int) assemble n p partStart sortedRows localGid globalBase canonOf nGroups = do rtgM <- VUM.new (max 1 n) - -- Global group id of each sorted position, plus per-group counts. counts <- VUM.replicate (nGroups + 1) (0 :: Int) gidAt <- VUM.new (max 1 n) let scanPos !pp @@ -308,7 +265,6 @@ assemble n p partStart sortedRows localGid globalBase canonOf nGroups = do inner s scanPos (pp + 1) scanPos 0 - -- offsets = exclusive prefix sum of counts. offsM <- VUM.new (nGroups + 1) let scan !k !acc | k > nGroups = pure () @@ -317,9 +273,6 @@ assemble n p partStart sortedRows localGid globalBase canonOf nGroups = do c <- if k < nGroups then VUM.unsafeRead counts k else pure 0 scan (k + 1) (acc + c) scan 0 0 - -- valueIndices: place each sorted position's row at its group's cursor. - -- Iterating sorted positions in order keeps rows in original order within a - -- group (the partition counting sort and grouping both preserve it). cursor <- VUM.new (max 1 nGroups) forM_ [0 .. nGroups - 1] $ \k -> VUM.unsafeRead offsM k >>= VUM.unsafeWrite cursor k visM <- VUM.new (max 1 n) diff --git a/dataframe-core/src/DataFrame/Internal/Hash.hs b/dataframe-core/src-internal/DataFrame/Internal/Hash.hs similarity index 64% rename from dataframe-core/src/DataFrame/Internal/Hash.hs rename to dataframe-core/src-internal/DataFrame/Internal/Hash.hs index 642b54b0..310d32f1 100644 --- a/dataframe-core/src/DataFrame/Internal/Hash.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Hash.hs @@ -2,14 +2,9 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE MagicHash #-} -{- | -A poor-man's hash used by 'DataFrame.Internal.Grouping' to bucket rows -without depending on the @hashable@ package. - -Each value is folded into an 'Int' accumulator with an FxHash-style step -(rotate, xor, multiply). It is intentionally small and not cryptographically -strong — it only needs to spread group-key tuples well enough that -'Data.IntMap' bucketing produces sensible groups. +{- | A poor-man's hash used by 'DataFrame.Internal.Grouping' to bucket rows +without depending on @hashable@. Each value is folded into an 'Int' with an +FxHash-style step (rotate, xor, multiply); small and not cryptographic. -} module DataFrame.Internal.Hash ( fnvOffset, @@ -47,23 +42,16 @@ fnvOffset = fromIntegral (0xcbf29ce484222325 :: Word64) fnvPrime :: Int fnvPrime = 0x00000100000001b3 -{- | Sentinel mixed in for a /null/ slot of a nullable column, so that a -@Nothing@ does not hash to the same value as a present @Just x@ that happens to -store the same underlying bits (notably @Just 0@). A fixed distinctive constant -(the 64-bit golden-ratio mix constant) keeps null hashing deterministic; a real -value equal to it collides only as rarely as any other hash collision. +{- | Sentinel mixed in for a /null/ slot, so @Nothing@ does not hash the same as +a present value with equal bits (e.g. @Just 0@). A fixed distinctive constant +keeps null hashing deterministic; a real value equal to it collides only rarely. -} nullSalt :: Int nullSalt = fromIntegral (0x9E3779B97F4A7C15 :: Word64) -{- | Mix an 'Int' into the accumulator. - -An FxHash-style step (rotate the accumulator, xor the value, multiply by a large -odd constant). The rotate diffuses each value's bits across all positions before -the next is folded in, so small/adjacent integers — common as group keys — do -not produce the structured collisions that a plain @(acc `xor` x) * prime@ does -once several columns are combined. Grouping trusts hash equality, so this -robustness is what keeps distinct rows in distinct groups. +{- | Mix an 'Int' into the accumulator with an FxHash-style step. The rotate +diffuses each value's bits before the next is folded in, avoiding the structured +collisions a plain xor-then-multiply produces on small/adjacent group keys. -} mixInt :: Int -> Int -> Int mixInt acc x = (rotateL acc 13 `xor` x) * fnvPrime @@ -84,12 +72,9 @@ mixChar :: Int -> Char -> Int mixChar acc = mixInt acc . ord {-# INLINE mixChar #-} -{- | Mix a 'T.Text' value into the accumulator over its raw UTF-8 bytes, -eight at a time. Reading a whole 'Word64' per step (rather than decoding and -mixing one codepoint at a time) cuts the multiply count ~8x on long keys while -staying collision-equivalent: UTF-8 is injective, so equal 'T.Text's mix to the -same value and distinct ones almost never collide. The trailing @len `mod` 8@ -bytes are folded in individually. +{- | Mix a 'T.Text' value into the accumulator over its raw UTF-8 bytes, eight at +a time. Reading a whole 'Word64' per step cuts the multiply count ~8x on long +keys while staying collision-equivalent (UTF-8 is injective). -} mixText :: Int -> T.Text -> Int mixText !acc (Text arr off len) = mixBytes acc arr off len diff --git a/dataframe-core/src/DataFrame/Internal/HashTable.hs b/dataframe-core/src-internal/DataFrame/Internal/HashTable.hs similarity index 63% rename from dataframe-core/src/DataFrame/Internal/HashTable.hs rename to dataframe-core/src-internal/DataFrame/Internal/HashTable.hs index 751212d0..ce2e612e 100644 --- a/dataframe-core/src/DataFrame/Internal/HashTable.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/HashTable.hs @@ -2,24 +2,9 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | -A flat, unboxed, open-addressing (linear-probe) hash table that maps a row's -key-hash to a /dense group id/, verifying the real key on every hash hit. - -The table is three parallel unboxed 'VUM.MVector's keyed by hash slot: - - * @htHash@ — the stored hash at each slot. - * @htGroup@ — the dense group id stored at each slot (@-1@ marks an empty - slot, since real group ids are @>= 0@). - * @htRep@ — the representative row index of that group, used to re-verify - the real key columns on a hash hit and so reject collisions. - -It is 'PrimMonad'-polymorphic: it runs in 'Control.Monad.ST.ST' for the current -single-threaded 'DataFrame.Internal.Grouping.groupBy' and can run in 'IO' inside -a per-worker partition once grouping is parallelised. The lookup-or-insert loop -('htInsert') trusts the caller-supplied @eqRow@ predicate to compare the key -columns of two rows by index, fixing the hash-only bucketing that previously -merged colliding keys. +{- | A flat, unboxed, open-addressing (linear-probe) hash table mapping a row's +key-hash to a dense group id, re-verifying the real key on every hash hit to +reject collisions. Runs in any 'PrimMonad' ('ST' for grouping, 'IO' per worker). -} module DataFrame.Internal.HashTable ( HashTable (..), @@ -67,13 +52,9 @@ newHashTable n = do pure (HashTable h g r (cap - 1)) {-# INLINE newHashTable #-} -{- | Look up @row@ (with precomputed @hash@) in the table, returning its dense -group id. On an empty slot the row starts a new group: the caller's -@nextGroup@ thunk supplies the next dense id, and the row is recorded as that -group's representative. On a stored-hash match the real key is re-verified with -@eqRow rep row@ before the existing id is returned; a mismatch is a hash -collision and probing continues. The returned 'Bool' is 'True' when a new group -was created, letting the caller bump its group counter without a second read. +{- | Look up @row@ (with precomputed @hash@) and return its dense group id: an +empty slot starts a new group via @nextGroup@, a stored-hash match is re-verified +with @eqRow@ before reuse. The 'Bool' is 'True' when a new group was created. -} htInsert :: (PrimMonad m) => diff --git a/dataframe-core/src/DataFrame/Internal/Interpreter.hs b/dataframe-core/src-internal/DataFrame/Internal/Interpreter.hs similarity index 92% rename from dataframe-core/src/DataFrame/Internal/Interpreter.hs rename to dataframe-core/src-internal/DataFrame/Internal/Interpreter.hs index f22fa1b4..46ba4ab8 100644 --- a/dataframe-core/src/DataFrame/Internal/Interpreter.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Interpreter.hs @@ -358,10 +358,9 @@ liftValue f (Flat col) = Flat <$> mapColumn f col liftValue f (Group gs) = Group <$> V.mapM (mapColumn f) gs {-# INLINEABLE liftValue #-} -{- | Apply a binary function to two 'Value's. When one side is a -'Scalar' the operation degenerates to a 'liftValue' — this is how the -old @Binary op (Lit l) right@ special cases are recovered without -explicit pattern matches in the evaluator. +{- | Apply a binary function to two 'Value's. When one side is a 'Scalar' the +operation degenerates to 'liftValue', recovering the old @Binary op (Lit l) right@ +special cases without explicit pattern matches. -} liftValue2 :: (Columnable c, Columnable b, Columnable a) => @@ -376,7 +375,6 @@ liftValue2 f (Flat l) (Flat r) = Flat <$> zipWithColumns f l r liftValue2 f (Group ls) (Group rs) | V.length ls == V.length rs = Group <$> V.zipWithM (zipWithColumns f) ls rs --- Shape mismatches: aggregated vs. non-aggregated. liftValue2 _ (Flat _) (Group _) = Left $ AggregatedAndNonAggregatedException "aggregated" "non-aggregated" liftValue2 _ (Group _) (Flat _) = @@ -458,7 +456,6 @@ enrichError loc (TypeMismatchException ctx) = errorColumnName ctx <|+> Just loc } where - -- Prefer the existing value; fall back to the new one. Nothing <|+> b = b a <|+> _ = a enrichError _ e = e @@ -511,16 +508,9 @@ invertPermutation perm = VU.create $ do -- promoteColumnWith: unified numeric / text coercion for CastWith ------------------------------------------------------------------------------- -{- | Apply a result-handler @onResult@ to each element of a column after -coercing it to type @a@. Covers three modes in one: - -* @onResult = either (const Nothing) Just@ → like @cast@ (returns @Maybe a@) -* @onResult = either (const def) id@ → like @castWithDefault@ (returns @a@) -* @onResult = either (Left . T.pack) Right@ → like @castEither@ (returns @Either T.Text a@) - -Numeric coercion handles Double, Float, and Int targets. Text columns -(String / T.Text) are parsed via 'reads'. Any other mismatch returns -'Left TypeMismatchException'. +{- | Coerce a column to type @a@, then apply @onResult@ to each element; the handler +selects the mode (like @cast@, @castWithDefault@, or @castEither@). Handles Double/ +Float/Int coercion and 'reads'-parses Text; other mismatches return 'Left'. -} promoteColumnWith :: forall a b. @@ -715,13 +705,9 @@ tryParseWith onResult col = case col of ) (V.convert v) -{- | When the output type @b@ is @Maybe c@ (or @Maybe (Maybe c)@) and the -column stores plain @c@ values, wrap each element in 'Just'. -The @Maybe (Maybe c)@ case applies join semantics: instead of producing -a double-wrapped column, a @Maybe c@ column is returned, so -@castExpr \@(Maybe Double)@ on a @Double@ column yields @Maybe Double@ -rather than @Maybe (Maybe Double)@. -Returns 'Nothing' when neither condition holds. +{- | When output type @b@ is @Maybe c@ (or @Maybe (Maybe c)@) and the column stores +plain @c@, wrap each element in 'Just' (the double-Maybe case collapses to a single +@Maybe c@). Returns 'Nothing' when neither condition holds. -} tryMaybeWrap :: forall a b. @@ -772,8 +758,6 @@ eval :: forall a. (Columnable a) => Ctx -> Expr a -> Either DataFrameException (Value a) --- Leaves ----------------------------------------------------------------- - eval _ (Lit v) = Right (Scalar v) eval (FlatCtx df) (Col name) = case getColumn name df of @@ -814,8 +798,6 @@ eval (GroupCtx gdf) (Col name) = } :: TypeErrorContext a () ) --- CastWith --------------------------------------------------------------- - eval (FlatCtx df) (CastWith name _tag onResult) = case getColumn name df of Nothing -> @@ -833,8 +815,6 @@ eval (GroupCtx gdf) (CastWith name _tag onResult) = Just c -> do promoted <- promoteColumnWith onResult c Right $ Group (sliceGroups promoted (offsets gdf) (valueIndices gdf)) --- CastExprWith ----------------------------------------------------------- - eval ctx (CastExprWith _tag onResult (inner :: Expr src)) = do v <- eval @src ctx inner case v of @@ -844,30 +824,19 @@ eval ctx (CastExprWith _tag onResult (inner :: Expr src)) = do Flat <$> promoteColumnWith onResult col Group gs -> Group <$> V.mapM (promoteColumnWith onResult) gs --- Unary ------------------------------------------------------------------ - eval ctx expr@(Unary op (inner :: Expr b)) = addContext expr $ do v <- eval @b ctx inner liftValue (unaryFn op) v - --- Binary ----------------------------------------------------------------- - eval ctx expr@(Binary op (left :: Expr c) (right :: Expr b)) = addContext expr $ do l <- eval @c ctx left r <- eval @b ctx right liftValue2 (binaryFn op) l r - --- If --------------------------------------------------------------------- - eval ctx expr@(If cond l r) = addContext expr $ do c <- eval @Bool ctx cond lv <- eval @a ctx l rv <- eval @a ctx r branchValue c lv rv - --- Over (window function) ------------------------------------------------- - eval (FlatCtx df) expr@(Over keys inner) = addContext expr $ do let gdf = G.groupBy keys df v <- eval (GroupCtx gdf) inner @@ -875,11 +844,8 @@ eval (FlatCtx df) expr@(Over keys inner) = addContext expr $ do Scalar s -> Right (Scalar s) Flat groupCol -> - -- Scalar agg (mean, sum, median): one value per group. - -- Broadcast via rowToGroup: row i gets value at group rowToGroup[i]. Right (Flat (atIndicesStable (rowToGroup gdf) groupCol)) Group groupCols -> do - -- Concatenate in sorted order, then unsort to original row order. sorted <- V.fold1M' concatColumns groupCols let inv = invertPermutation (valueIndices gdf) Right (Flat (atIndicesStable inv sorted)) @@ -945,8 +911,6 @@ eval <$> ( foldLinearGroups @b step seed col (rowToGroup gdf) (numGroups gdf) >>= mapColumn finalize ) --- Aggregation: CollectAgg ------------------------------------------------ - eval ctx expr@(Agg (CollectAgg _ (f :: v b -> a)) inner) = addContext expr $ do v <- eval @b ctx inner @@ -960,9 +924,6 @@ eval ctx expr@(Agg (CollectAgg _ (f :: v b -> a)) inner) = Group gs -> Flat . fromVector <$> V.mapM (applyCollect @v @b @a f) gs - --- Aggregation: FoldAgg with seed ----------------------------------------- - eval ctx expr@(Agg (FoldAgg _ (Just seed) (f :: a -> b -> a)) inner) = addContext expr $ do v <- eval @b ctx inner @@ -973,9 +934,6 @@ eval ctx expr@(Agg (FoldAgg _ (Just seed) (f :: a -> b -> a)) inner) = Group gs -> Flat . fromVector <$> V.mapM (foldlColumn @b @a f seed) gs - --- Aggregation: MergeAgg -------------------------------------------------- - eval ctx expr@( Agg @@ -998,9 +956,6 @@ eval Group gs -> Flat . fromVector <$> V.mapM (fmap finalize . foldlColumn @b step seed) gs - --- Aggregation: FoldAgg without seed (fold1) ------------------------------ - eval ctx expr@(Agg (FoldAgg _ Nothing (f :: a -> b -> a)) inner) = addContext expr $ case testEquality (typeRep @a) (typeRep @b) of @@ -1060,13 +1015,9 @@ data AggregationResult a = UnAggregated Column | Aggregated (TypedColumn a) -{- | Interpret an expression against a flat 'DataFrame', producing a -typed column. This is the original top-level entry point; internally -it calls 'eval' and materialises the result. - -NOTE: unlike the old implementation, 'Lit' values are no longer -eagerly broadcast. The broadcast happens here, at the boundary, -via 'materialize'. +{- | Interpret an expression against a flat 'DataFrame', producing a typed column. +Calls 'eval' then 'materialize'; 'Lit' values are broadcast here at the boundary +rather than eagerly. -} interpret :: forall a. @@ -1097,7 +1048,4 @@ interpretAggregation gdf expr = do Flat col -> Right $ Aggregated $ TColumn col Group _ -> - -- The Column payload is intentionally unused — the only - -- call-site ('aggregate') immediately throws - -- 'UnaggregatedException' on this constructor. Right $ UnAggregated $ BoxedColumn @T.Text Nothing V.empty diff --git a/dataframe-core/src/DataFrame/Internal/Nullable.hs b/dataframe-core/src-internal/DataFrame/Internal/Nullable.hs similarity index 87% rename from dataframe-core/src/DataFrame/Internal/Nullable.hs rename to dataframe-core/src-internal/DataFrame/Internal/Nullable.hs index d4af803c..04ac1441 100644 --- a/dataframe-core/src/DataFrame/Internal/Nullable.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Nullable.hs @@ -9,24 +9,9 @@ {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE UndecidableSuperClasses #-} -{- | Nullable-aware binary operations for expressions. - -This module provides two type classes, 'NullableArithOp' and 'NullableCmpOp', -which enable operators like '.+', '.-', '.*', './', '.==' etc. to work -transparently across combinations of nullable (@Maybe a@) and non-nullable -(@a@) column types. - -The partial functional dependencies uniquely determine the result type from -the operand types, so GHC infers it without annotations. - -The four combinations covered for each class: - -* @(a, a)@ — non-nullable × non-nullable -* @(Maybe a, a)@ — nullable × non-nullable -* @(a, Maybe a)@ — non-nullable × nullable -* @(Maybe a, Maybe a)@ — both nullable - -== Usage +{- | Nullable-aware arithmetic and comparison operators ('.+', '.==', …) that work +transparently across nullable (@Maybe a@) and non-nullable (@a@) operands. +Functional dependencies infer the result type without annotations. @ -- Mixing nullable and non-nullable columns: @@ -87,13 +72,9 @@ type family BaseType a where BaseType (Maybe a) = a BaseType a = a -{- | Class for arithmetic binary operations that work transparently over -nullable and non-nullable column types. - -The functional dependency @a b -> c@ ensures GHC can infer the result type @c@ -from the operand types. The 'OVERLAPPABLE' pragma on the non-nullable instance -ensures the more specific @(Maybe a, Maybe a)@ instance wins when both operands -are nullable. +{- | Arithmetic binary operations that work over nullable and non-nullable operand +types. The functional dependency @a b -> c@ infers the result; the 'OVERLAPPABLE' +non-nullable instance yields to the specific @(Maybe a, Maybe a)@ one. -} class ( Columnable a @@ -128,13 +109,9 @@ type family NullCmpResult a b where NullCmpResult a (Maybe b) = Maybe Bool NullCmpResult a b = Bool -{- | Class for comparison binary operations that work transparently over -nullable and non-nullable column types. - -No functional dependency on @e@: the 'OVERLAPPING'\/'OVERLAPPABLE' pragmas on -instances disambiguate at call sites without a FundDep (which would conflict -when both operands are @Maybe@). GHC selects the unique most-specific instance -from the concrete operand types. +{- | Comparison binary operations over nullable and non-nullable operands. No +functional dependency on @e@; overlapping/overlappable instance pragmas pick the +unique most-specific instance from the concrete operand types. -} class ( Columnable a @@ -229,12 +206,9 @@ instance -- Generalized nullable lift (unary) -- --------------------------------------------------------------------------- -{- | Lift a unary function over a column expression, propagating 'Nothing'. - -When @a@ is non-nullable the function is applied directly; when @a = Maybe x@ -the function is applied under the 'Just' and 'Nothing' short-circuits. - -Use via 'DataFrame.Functions.nullLift'. +{- | Lift a unary function over a column expression, propagating 'Nothing' (applied +directly when non-nullable, under 'Just' when @a = Maybe x@). Use via +'DataFrame.Functions.nullLift'. -} {- | Compute the result type of a nullable unary lift. @@ -281,16 +255,9 @@ instance -- Generalized nullable lift (binary) -- --------------------------------------------------------------------------- -{- | Lift a binary function over two column expressions, propagating 'Nothing'. - -The four combinations: - -* @(a, b)@ — both non-nullable: result is @r@ -* @(Maybe a, b)@ — left nullable: result is @Maybe r@ -* @(a, Maybe b)@ — right nullable: result is @Maybe r@ -* @(Maybe a, Maybe b)@ — both nullable: result is @Maybe r@ - -Use via 'DataFrame.Functions.nullLift2'. +{- | Lift a binary function over two column expressions, propagating 'Nothing': the +result is @Maybe r@ if either operand is nullable, else @r@. Use via +'DataFrame.Functions.nullLift2'. -} {- | Compute the result type of a nullable binary lift. diff --git a/dataframe-core/src/DataFrame/Internal/PackedText.hs b/dataframe-core/src-internal/DataFrame/Internal/PackedText.hs similarity index 70% rename from dataframe-core/src/DataFrame/Internal/PackedText.hs rename to dataframe-core/src-internal/DataFrame/Internal/PackedText.hs index 5a685238..d26285c8 100644 --- a/dataframe-core/src/DataFrame/Internal/PackedText.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/PackedText.hs @@ -1,15 +1,8 @@ {-# LANGUAGE BangPatterns #-} -{- | Packed-text payload + byte-slice primitives. A 'PackedTextData' shares a -single UTF-8 byte buffer across all rows of a string column, with @n+1@ row -offsets, so no per-row 'Data.Text.Text' header is materialized at freeze. -'Data.Text.Text' is produced only on demand (display, typed extraction) via -the same decode path that the boxed-Text builder used. - -A gathered/joined/sorted result keeps sharing that buffer: instead of copying -bytes it carries a @ptSel@ selection vector that reindexes the base rows, so a -permuted or row-exploded column stays a 'PackedText' (shared buffer + permuted -indices) rather than materializing back to boxed 'Data.Text.Text'. +{- | Packed-text payload + byte-slice primitives. A 'PackedTextData' shares one +UTF-8 byte buffer across all rows of a string column, with @n+1@ row offsets, so +no per-row 'Data.Text.Text' header is materialized until decode is demanded. -} module DataFrame.Internal.PackedText ( PackedTextData (..), @@ -33,14 +26,8 @@ import Data.Text.Internal (Text (Text)) import DataFrame.Internal.Utf8 (isValidUtf8Slice, lenientDecodeSlice) {- | A shared UTF-8 byte buffer plus @n+1@ row offsets (base row @r@ spans bytes -@[offsets!r, offsets!(r+1))@). Validity lives in the enclosing column's -@Maybe Bitmap@, mirroring 'BoxedColumn'/'UnboxedColumn'. - -@ptSel@ is an optional selection layer: when @Nothing@ the column is the -contiguous base (row @i@ == base row @i@). When @Just sel@, logical row @i@ is -base row @sel!i@; this is how a gather/join/sort result shares the buffer -without copying bytes. Out-of-range entries in @sel@ (e.g. a join @-1@ -sentinel) decode to the empty slice and are masked by the column bitmap. +@[offsets!r, offsets!(r+1))@); validity lives in the column's bitmap. @ptSel@ is +an optional selection layer letting a gather/join/sort result share the buffer. -} data PackedTextData = PackedTextData { ptBytes :: {-# UNPACK #-} !A.Array @@ -53,11 +40,9 @@ mkPackedContiguous :: A.Array -> VU.Vector Int -> PackedTextData mkPackedContiguous arr offs = PackedTextData arr offs Nothing {-# INLINE mkPackedContiguous #-} -{- | Reindex a packed payload by a selection vector, sharing the byte buffer -and base offsets. Logical row @i@ becomes base row @indices!i@. A negative or -out-of-range index decodes to the empty slice (callers mask it with a bitmap). -Composes with an existing selection so a gather of a gather still shares the -buffer. +{- | Reindex a packed payload by a selection vector, sharing the byte buffer; +logical row @i@ becomes base row @indices!i@. A negative or out-of-range index +decodes to the empty slice. Composes with an existing selection. -} packedGather :: VU.Vector Int -> PackedTextData -> PackedTextData packedGather indices (PackedTextData arr offs msel) = @@ -72,12 +57,9 @@ packedGather indices (PackedTextData arr offs msel) = in PackedTextData arr offs (Just sel') {-# INLINE packedGather #-} -{- | Take the first @k@ logical rows, sharing the byte buffer. With a selection -layer the selection is sliced to @k@ entries; without one a base-row selection -@[0 .. k-1]@ is installed (slicing the contiguous offsets would still leave the -trailing bytes addressable, but a short selection caps 'packedLength' to @k@). -O(k), no byte copy or decode — the fix for cheap @take@/display on a 1e7-row -packed column. +{- | Take the first @k@ logical rows, sharing the byte buffer via a capped +selection layer. O(k), no byte copy or decode — cheap @take@/display on a +large packed column. -} packedTake :: Int -> PackedTextData -> PackedTextData packedTake k (PackedTextData arr offs msel) = @@ -111,10 +93,8 @@ packedSlice p@(PackedTextData arr offs _) i = {-# INLINE packedSlice #-} {- | The shared buffer + contiguous @n+1@ offsets when the payload is the -unselected base. A selected (gathered) payload has non-contiguous rows that a -single offset vector cannot express, so this returns @Nothing@ and the caller -decodes per-row via 'packedIndexText'. Lets the boxed-Text fallback take the -fast contiguous 'sliceTextVector' path when possible. +unselected base; a selected (gathered) payload returns 'Nothing' (its rows are +non-contiguous). Lets the boxed-Text fallback take the fast contiguous path. -} packedRowOffsetVec :: PackedTextData -> Maybe (A.Array, VU.Vector Int) packedRowOffsetVec (PackedTextData arr offs Nothing) = Just (arr, offs) diff --git a/dataframe-core/src/DataFrame/Internal/ParRadixSort.hs b/dataframe-core/src-internal/DataFrame/Internal/ParRadixSort.hs similarity index 86% rename from dataframe-core/src/DataFrame/Internal/ParRadixSort.hs rename to dataframe-core/src-internal/DataFrame/Internal/ParRadixSort.hs index c0a703fe..d71e4472 100644 --- a/dataframe-core/src/DataFrame/Internal/ParRadixSort.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/ParRadixSort.hs @@ -1,27 +1,9 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | -Parallel stable sort of row indices by the ascending unsigned order of a -per-row 'Int' hash. Shared by the join build-side 'CompactIndex' construction -(@DataFrame.Operations.Join@), which previously paid a single-threaded -comparison sort over the whole build side — the dominant serial cost of a large -inner join (the @1e7 x 1e7@ big-inner case). - -@parSortByHash n hashes@ returns @(sortedHashes, sortedIndices)@ where -@sortedIndices@ lists @[0, n)@ in ascending 'sortKey' order of their hash, ties -broken by ascending original index (stable), and @sortedHashes[k] == -hashes[sortedIndices[k]]@. Bit-for-bit identical to the old stable merge sort's -output ordering, so equal-hash rows stay contiguous (the run scan in -'buildCompactIndex' depends on this) and within a run keep original-row order. - -Strategy (mirrors "DataFrame.Internal.GroupingPar"): a counting sort buckets -rows by the top @log2 p@ bits of their unsigned key into @p@ partitions laid out -in ascending key order; @caps@ 'forkIO' workers then LSD-radix-sort each -partition by the full 56 remaining low bits. Because partitions are already in -global key order and each per-partition sort is stable, concatenating them -reproduces the global stable order with no merge step. A sequential LSD radix -sort is used below 'parSortThreshold' or on a single capability. +{- | Parallel stable sort of row indices by ascending unsigned order of a per-row +'Int' hash, used by the join build side. A counting sort buckets rows into +key-ordered partitions that workers LSD-radix-sort in parallel, with no merge step. -} module DataFrame.Internal.ParRadixSort ( parSortByHash, @@ -174,11 +156,7 @@ parSortByHashIO n hashes = do caps <- getNumCapabilities let !p = numPartitionsFor caps !shift = 64 - intLog2 p - -- Phase 1: counting sort of row indices into ascending-key partitions. (partStart, partRows) <- partitionRows n hashes p shift - -- Phase 2: stable-sort each partition by full key, in parallel. Each worker - -- owns disjoint [partStart[pp], partStart[pp+1]) output ranges, so the - -- single shared output buffers are written race-free. outOrder <- VUM.new n outKeys <- VUM.new n sortPartitions caps p partStart partRows hashes outOrder outKeys diff --git a/dataframe-core/src/DataFrame/Internal/Pretty.hs b/dataframe-core/src-internal/DataFrame/Internal/Pretty.hs similarity index 80% rename from dataframe-core/src/DataFrame/Internal/Pretty.hs rename to dataframe-core/src-internal/DataFrame/Internal/Pretty.hs index d31e89f4..35f4ecd2 100644 --- a/dataframe-core/src/DataFrame/Internal/Pretty.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Pretty.hs @@ -1,10 +1,6 @@ {- | A minimal Wadler/Leijen-style document combinator and width-aware renderer. - -This is a small self-contained port of the "prettiest printer" algorithm -(Wadler, "A prettier printer"; Leijen, @wl-pprint@). A 'Doc' describes a -layout abstractly; 'render' chooses where soft breaks ('line') turn into -newlines so the result fits a target width. 'Group' marks a region that -should be laid out flat when it fits, broken otherwise. +A 'Doc' describes a layout abstractly; 'render' chooses where soft breaks become +newlines to fit a target width. 'Group' lays a region flat when it fits. -} module DataFrame.Internal.Pretty ( Doc, @@ -78,10 +74,9 @@ punctuate sep (d : ds) = (d <> sep) : punctuate sep ds parens :: Doc -> Doc parens d = Text "(" <> d <> Text ")" -{- | Render @d@ bare when it fits flat on the current line; wrap it in -parentheses when it must break across multiple lines. Used to keep operator -grouping unambiguous once a sub-expression wraps, without adding parenthesis -noise to expressions that stay on one line. +{- | Render @d@ bare when it fits flat on the current line, wrapped in parens when +it must break across lines. Keeps operator grouping unambiguous once a +sub-expression wraps, without parenthesis noise on one-line expressions. -} parensWhenBroken :: Doc -> Doc parensWhenBroken d = Group (Alt d (parens d)) @@ -125,9 +120,6 @@ render width doc = layout 0 [(0, Break, doc)] Line -> case m of Flat -> fits (w - 1) rest Break -> True - -- A forced break inside the group being measured (Flat) means it cannot - -- lay out on one line; in the surrounding context (Break) it just ends - -- the current line, so the prefix so far fits. Hard -> case m of Flat -> False Break -> True diff --git a/dataframe-core/src/DataFrame/Internal/RadixRank.hs b/dataframe-core/src-internal/DataFrame/Internal/RadixRank.hs similarity index 77% rename from dataframe-core/src/DataFrame/Internal/RadixRank.hs rename to dataframe-core/src-internal/DataFrame/Internal/RadixRank.hs index 29e0d013..c4547dd0 100644 --- a/dataframe-core/src/DataFrame/Internal/RadixRank.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/RadixRank.hs @@ -1,18 +1,9 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | -Stable rank of a set of group representatives by the ascending unsigned order of -their hash. Shared by the sequential ('DataFrame.Internal.Grouping') and parallel -('DataFrame.Internal.GroupingPar') group-by canonical-ordering steps so they -stay bit-for-bit identical. - -@rankByHash readHash ng@ returns @rank@ with @rank[gid] = position@ of group -@gid@ when groups are ordered by ascending unsigned 'sortKey' of @readHash gid@. -A stable LSD radix sort (8 bits per pass, 8 passes) keeps groups with equal -hash in their original @gid@ order; callers number @gid@s so that this matches -the @repRow@ tie-break of the old comparison sort. @O(ng)@, no boxed tuples or -comparison closures — the lever for the @1e7@-distinct-group case (Q10). +{- | Stable rank of a set of group representatives by ascending unsigned hash +order. Shared by the sequential and parallel group-by canonical-ordering steps +so they stay bit-for-bit identical. @O(ng)@ stable LSD radix sort. -} module DataFrame.Internal.RadixRank ( rankByHash, @@ -27,9 +18,8 @@ import qualified Data.Vector.Unboxed.Mutable as VUM import Data.Word (Word64) {- | Unsigned sort key of a hash: ascending 'Word64' order of @sortKey h@ equals -ascending signed-'Int' order of @h@. Reinterpreted back to 'Int' for the -byte-wise radix passes (the @.&. 0xff@ byte mask makes the arithmetic shift's -sign extension irrelevant). +ascending signed-'Int' order of @h@. Reinterpreted to 'Int' for the byte-wise +radix passes (the byte mask makes the sign extension irrelevant). -} sortKey :: Int -> Int sortKey h = fromIntegral (fromIntegral h + 0x8000000000000000 :: Word64) @@ -92,8 +82,6 @@ rankByHash readHash ng = do VUM.unsafeWrite dstO pos o place (i + 1) place 0 - -- 8 stable passes over the 64-bit key; ping-pong so the final - -- sorted order lands back in (keysA, orderA). pass 0 keysA orderA keysB orderB pass 8 keysB orderB keysA orderA pass 16 keysA orderA keysB orderB @@ -102,7 +90,6 @@ rankByHash readHash ng = do pass 40 keysB orderB keysA orderA pass 48 keysA orderA keysB orderB pass 56 keysB orderB keysA orderA - -- orderA[rank] = gid; invert to rank[gid] = rank. let inv !r | r >= ng = pure () | otherwise = do diff --git a/dataframe-core/src/DataFrame/Internal/Row.hs b/dataframe-core/src-internal/DataFrame/Internal/Row.hs similarity index 79% rename from dataframe-core/src/DataFrame/Internal/Row.hs rename to dataframe-core/src-internal/DataFrame/Internal/Row.hs index e54453e6..8e31b59e 100644 --- a/dataframe-core/src/DataFrame/Internal/Row.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Row.hs @@ -81,10 +81,8 @@ type Row = V.Vector Any (!?) (_x : xs) n = (!?) xs (n - 1) {- | Reconstruct column @i@ from a list of rows. The element type is taken from -the first non-'Null' cell; cells of a different type are skipped. If any cell is -'Null' the result is a nullable column (built with 'fromMaybeVec', which needs -only @Columnable a@), so a round-trip through 'toRowList'/'fromRows' preserves -nulls. +the first non-'Null' cell; a differently-typed cell is skipped. If any cell is +'Null' the result is a nullable column, so a round-trip preserves nulls. -} mkColumnFromRow :: Int -> [[Any]] -> Column mkColumnFromRow i rows = @@ -101,26 +99,17 @@ mkColumnFromRow i rows = in if any isNothing maybes then fromMaybeVec (V.fromList maybes) else fromList (catMaybes maybes) - Just Null -> fromList ([] :: [T.Text]) -- unreachable: find isValue + Just Null -> fromList ([] :: [T.Text]) where isValue (Value _) = True isValue Null = False -{- | Converts the entire dataframe to a list of rows. - -Each row contains all columns in the dataframe, ordered by their column indices. -The rows are returned in their natural order (from index 0 to n-1). - -==== __Examples__ +{- | Convert the whole dataframe to a list of rows, one per row index in natural +order; each row lists all columns ordered by column index. Materializes every +row, so prefer 'toRowVector' for large frames. >>> toRowList df [[("name", "Alice"), ("age", 25), ...], [("name", "Bob"), ("age", 30), ...], ...] - -==== __Performance note__ - -This function materializes all rows into a list, which may be memory-intensive -for large dataframes. Consider using 'toRowVector' if you need random access -or streaming operations. -} toRowList :: DataFrame -> [[(T.Text, Any)]] toRowList df = @@ -131,26 +120,11 @@ toRowList df = (zip names . V.toList . mkRowRep df names) [0 .. (fst (dataframeDimensions df) - 1)] -{- | Converts the dataframe to a vector of rows with only the specified columns. - -Each row will contain only the columns named in the @names@ parameter. -This is useful when you only need a subset of columns or want to control -the column order in the resulting rows. - -==== __Parameters__ - -[@names@] List of column names to include in each row. The order of names - determines the order of fields in the resulting rows. - -[@df@] The dataframe to convert. - -==== __Examples__ +{- | Convert the dataframe to a vector of rows containing only the named columns, +in the given order. An empty name list yields one empty row per dataframe row. >>> toRowVector ["name", "age"] df Vector of rows with only name and age fields - ->>> toRowVector [] df -- Empty column list -Vector of empty rows (one per dataframe row) -} toRowVector :: [T.Text] -> DataFrame -> V.Vector Row toRowVector names df = V.generate (fst (dataframeDimensions df)) (mkRowRep df names) @@ -180,10 +154,8 @@ mkRowFromArgs names df i = V.map get (V.fromList names) Just (UnboxedColumn bm column) -> cellAny bm i (column VU.! i) Just (PackedText bm p) -> cellAny bm i (packedIndexText p i) --- This function will return the items in the order that is specified --- by the user. For example, if the dataframe consists of the columns --- "Age", "Pclass", "Name", and the user asks for ["Name", "Age"], --- this will order the values in the order ["Mr Smith", 50] +-- Returns row values in the caller's requested column order, not the +-- dataframe's storage order. mkRowRep :: DataFrame -> [T.Text] -> Int -> Row mkRowRep df names i = V.generate (L.length names) (\index -> get (names' V.! index)) where diff --git a/dataframe-core/src/DataFrame/Internal/RowHash.hs b/dataframe-core/src-internal/DataFrame/Internal/RowHash.hs similarity index 79% rename from dataframe-core/src/DataFrame/Internal/RowHash.hs rename to dataframe-core/src-internal/DataFrame/Internal/RowHash.hs index 0f4dcef2..800f541e 100644 --- a/dataframe-core/src/DataFrame/Internal/RowHash.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/RowHash.hs @@ -5,19 +5,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Row-hash kernels with a parallel driver. - -The per-row key hash is the sole input to grouping and the join build/probe. -For a single wide pass over many rows (notably a 1e7-row text/factor join key) -the hashing is the dominant cost and is embarrassingly parallel: each row's hash -depends only on that row's own bytes, so hashing disjoint row ranges into -disjoint slots of one shared vector is race-free and produces a result -/bit-for-bit identical/ to the sequential single-pass hash. - -'hashRowRange' is the shared per-range kernel (used sequentially and by every -worker); 'computeRowHashesIO' forks one worker per capability over contiguous -row ranges above 'parRowHashThreshold', else runs the range once. The mixing per -column type mirrors the grouping hash exactly so grouping and joins agree. +{- | Row-hash kernels with a parallel driver, feeding grouping and the join +build/probe. Each row's hash depends only on its own bytes, so hashing disjoint +ranges in parallel is race-free and bit-identical to the sequential pass. -} module DataFrame.Internal.RowHash ( computeRowHashesIO, @@ -67,12 +57,9 @@ capabilities :: Int capabilities = unsafePerformIO getNumCapabilities {-# NOINLINE capabilities #-} -{- | Compute the per-row key hash over the (already selected) key columns of an -@n@-row frame. Forks one worker per capability over contiguous row ranges when -the row count justifies it (>= 'parRowHashThreshold' and more than one -capability); otherwise hashes the single full range. The output is identical for -any capability count: each row's hash is a pure function of its own bytes and -workers own disjoint row ranges. +{- | Compute the per-row key hash over the selected key columns of an @n@-row +frame. Forks one worker per capability over disjoint row ranges when the row +count justifies it, else hashes the single full range; output is capability-independent. -} computeRowHashesIO :: Int -> [Column] -> IO (VU.Vector Int) computeRowHashesIO n selected = do @@ -95,9 +82,8 @@ computeRowHashesIO n selected = do VU.unsafeFreeze (VUM.slice 0 n mv) {- | Mix every selected column over the row range @[lo, hi)@ into @mv@, seeding -each slot with 'fnvOffset' first. The seeding and per-column mixing must match -'DataFrame.Operations.Aggregation.computeRowHashes' byte-for-byte so grouping -and joins bucket identically. +each slot with 'fnvOffset'. Must match the sequential grouping hash byte-for-byte +so grouping and joins bucket identically. -} hashRowRange :: VUM.IOVector Int -> Int -> Int -> [Column] -> IO () hashRowRange mv lo hi cols = do @@ -186,10 +172,8 @@ boxedRange mv lo hi bm mix v = go lo {-# INLINE boxedRange #-} {- | Mix a packed-text column's range over its raw UTF-8 byte slices. The -contiguous (unselected) payload is the hot path: hoist the byte buffer and the -@n+1@ offset vector out of the loop and index them directly, so each row mixes -@[offs!i, offs!(i+1))@ with no per-row selection 'Maybe' test or 'packedSlice' -tuple. A selected payload (a gather/join result) falls back to 'packedSlice'. +unselected payload is the hot path (indexes the offset vector directly); a +selected payload (a gather/join result) falls back to 'packedSlice'. -} packedRange :: VUM.IOVector Int -> diff --git a/dataframe-core/src/DataFrame/Internal/Simplify.hs b/dataframe-core/src-internal/DataFrame/Internal/Simplify.hs similarity index 97% rename from dataframe-core/src/DataFrame/Internal/Simplify.hs rename to dataframe-core/src-internal/DataFrame/Internal/Simplify.hs index 22d674a9..b32cb638 100644 --- a/dataframe-core/src/DataFrame/Internal/Simplify.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Simplify.hs @@ -410,13 +410,8 @@ factImplies (fc, ft) (cc, tc) where fIncl = fc == CGeq || fc == CLeq cIncl = cc == CGeq || cc == CLeq - -- same-direction containment: strictly tighter, or equal threshold where the - -- fact's boundary inclusivity is no stronger than the condition's. subset = (if isLower fc then ft > tc else ft < tc) || (ft == tc && (not fIncl || cIncl)) - -- lower fact ∩ upper cond empty: fact starts above cond's top, or they meet - -- at a point that is not in both. disjointAtEq = ft > tc || (ft == tc && not (fIncl && cIncl)) - -- upper fact ∩ lower cond empty (mirror). disjointBelow = ft < tc || (ft == tc && not (fIncl && cIncl)) diff --git a/dataframe-core/src/DataFrame/Internal/Types.hs b/dataframe-core/src-internal/DataFrame/Internal/Types.hs similarity index 96% rename from dataframe-core/src/DataFrame/Internal/Types.hs rename to dataframe-core/src-internal/DataFrame/Internal/Types.hs index e482e469..ce1da915 100644 --- a/dataframe-core/src/DataFrame/Internal/Types.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Types.hs @@ -93,7 +93,7 @@ class SBoolI (b :: Bool) where instance SBoolI 'True where sbool = STrue instance SBoolI 'False where sbool = SFalse --- | Type-level function to determine whether or not a type is unboxa +-- | Runtime witness for whether @a@ is unboxable. sUnbox :: forall a. (SBoolI (Unboxable a)) => SBool (Unboxable a) sUnbox = sbool @(Unboxable a) @@ -102,7 +102,7 @@ sNumeric = sbool @(Numeric a) type family When (flag :: Bool) (c :: Constraint) :: Constraint where When 'True c = c - When 'False c = () -- empty constraint + When 'False c = () type UnboxIf a = When (Unboxable a) (VU.Unbox a) @@ -158,4 +158,4 @@ type family PromoteDiv (a :: Type) (b :: Type) :: Type where PromoteDiv _ Double = Double PromoteDiv Float _ = Float PromoteDiv _ Float = Float - PromoteDiv _ _ = Double -- Int/Int32/Int64 in any combination + PromoteDiv _ _ = Double diff --git a/dataframe-core/src/DataFrame/Internal/Utf8.hs b/dataframe-core/src-internal/DataFrame/Internal/Utf8.hs similarity index 90% rename from dataframe-core/src/DataFrame/Internal/Utf8.hs rename to dataframe-core/src-internal/DataFrame/Internal/Utf8.hs index f1776e4a..62ba486b 100644 --- a/dataframe-core/src/DataFrame/Internal/Utf8.hs +++ b/dataframe-core/src-internal/DataFrame/Internal/Utf8.hs @@ -60,11 +60,8 @@ lenientDecodeSlice arr off len = T.pack (go off) | otherwise = loop (utf8DecodeContinue (A.unsafeIndex arr j) st cp) (j + 1) {- | Slice forced 'Text' values off a shared array; row @i@ spans bytes -@[offs!i, offs!(i+1))@. The offsets need not start at byte 0, so a row -sub-range of a larger offset vector slices independently (parallel text -merging uses this). Fast path: validate the spanned bytes once and check -every field starts on a code-point boundary. Slow path: per-field -validation with lenient decoding of invalid fields. +@[offs!i, offs!(i+1))@. Fast path validates the whole span once when every field +starts on a code-point boundary; else per-field validation with lenient decode. -} sliceTextVector :: A.Array -> VU.Vector Int -> VB.Vector T.Text sliceTextVector arr offs = VB.create $ do diff --git a/dataframe-core/src/DataFrame/Operators.hs b/dataframe-core/src-internal/DataFrame/Operators.hs similarity index 100% rename from dataframe-core/src/DataFrame/Operators.hs rename to dataframe-core/src-internal/DataFrame/Operators.hs diff --git a/dataframe-core/src/DataFrame/Core.hs b/dataframe-core/src/DataFrame/Core.hs new file mode 100644 index 00000000..48ed545a --- /dev/null +++ b/dataframe-core/src/DataFrame/Core.hs @@ -0,0 +1,104 @@ +{- | The curated public surface of @dataframe-core@: the interchange types +('DataFrame', 'Column', 'Row', 'Expr'), element constraints, and the +rendering/serialization verbs. Internal plumbing stays in @dataframe-core:internal@. +-} +module DataFrame.Core ( + -- * The DataFrame + DataFrame, + GroupedDataFrame, + empty, + fromNamedColumns, + insertColumn, + columnNames, + null, + + -- * Columns + Column, + fromList, + fromVector, + fromUnboxedVector, + mkRandom, + toList, + toVector, + hasElemType, + hasMissing, + isNumeric, + + -- * Element constraints + Columnable, + Columnable', + + -- * Rows + Row, + Any, + toAny, + fromAny, + rowValue, + toRowList, + toRowVector, + + -- * Expressions + Expr, + NamedExpr, + eSize, + prettyPrint, + prettyPrintWidth, + + -- * Rendering & serialization + TruncateConfig (..), + defaultTruncateConfig, + toCsv, + toCsv', + toSeparated, + toMarkdown, + toMarkdown', +) where + +import Prelude hiding (null) + +import DataFrame.Internal.Column ( + Column, + Columnable, + fromList, + fromUnboxedVector, + fromVector, + hasElemType, + hasMissing, + isNumeric, + mkRandom, + toList, + toVector, + ) +import DataFrame.Internal.DataFrame ( + DataFrame, + GroupedDataFrame, + TruncateConfig (..), + columnNames, + defaultTruncateConfig, + empty, + fromNamedColumns, + insertColumn, + null, + toCsv, + toCsv', + toMarkdown, + toMarkdown', + toSeparated, + ) +import DataFrame.Internal.Expression ( + Expr, + NamedExpr, + eSize, + prettyPrint, + prettyPrintWidth, + ) +import DataFrame.Internal.Row ( + Any, + Row, + fromAny, + rowValue, + toAny, + toRowList, + toRowVector, + ) +import DataFrame.Internal.Types (Columnable') diff --git a/dataframe-csv-th/dataframe-csv-th.cabal b/dataframe-csv-th/dataframe-csv-th.cabal index 54cc219c..323d0274 100644 --- a/dataframe-csv-th/dataframe-csv-th.cabal +++ b/dataframe-csv-th/dataframe-csv-th.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe-csv-th -version: 1.0.1.1 +version: 1.1.0.0 synopsis: CSV-file-based Template Haskell splices for the dataframe ecosystem. description: @@ -31,8 +31,8 @@ library DataFrame.TH.CSV DataFrame.Typed.TH.CSV build-depends: base >= 4 && < 5, - dataframe-csv ^>= 1.0.2, - dataframe-th ^>= 1.0, + dataframe-csv >= 2.0 && < 2.1, + dataframe-th >= 2.0 && < 2.1, template-haskell >= 2.0 && < 3 hs-source-dirs: src default-language: Haskell2010 diff --git a/dataframe-csv/dataframe-csv.cabal b/dataframe-csv/dataframe-csv.cabal index e26fa631..8d48c353 100644 --- a/dataframe-csv/dataframe-csv.cabal +++ b/dataframe-csv/dataframe-csv.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-csv -version: 1.0.3.0 +version: 2.0.0.0 synopsis: CSV reader and writer for the dataframe ecosystem. description: @DataFrame.IO.CSV@ — strict single-pass CSV read/write (pure @@ -24,23 +24,37 @@ common warnings -Wunused-local-binds -Wunused-packages +library internal + import: warnings + visibility: public + exposed-modules: DataFrame.IO.Internal.MutableColumn + build-depends: base >= 4 && < 5, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, + text >= 2.1 && < 3, + vector ^>= 0.13 + hs-source-dirs: src-internal + default-language: Haskell2010 + library import: warnings exposed-modules: DataFrame.IO.CSV + DataFrame.Typed.IO.CSV + other-modules: DataFrame.IO.CSV.Internal.Infer DataFrame.IO.CSV.Internal.Options DataFrame.IO.CSV.Internal.Read DataFrame.IO.CSV.Internal.Scanner DataFrame.IO.CSV.Internal.Sink - DataFrame.IO.Internal.MutableColumn - DataFrame.Typed.IO.CSV build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9, - dataframe-core ^>= 1.1.1, - dataframe-operations ^>= 1.1.2, - dataframe-parsing ^>= 1.0.2, + dataframe-core >= 2.0 && < 2.1, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-operations >= 2.0 && < 2.1, + dataframe-parsing >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, text >= 2.1 && < 3, time >= 1.12 && < 2, vector ^>= 0.13 diff --git a/dataframe-csv/src/DataFrame/IO/Internal/MutableColumn.hs b/dataframe-csv/src-internal/DataFrame/IO/Internal/MutableColumn.hs similarity index 100% rename from dataframe-csv/src/DataFrame/IO/Internal/MutableColumn.hs rename to dataframe-csv/src-internal/DataFrame/IO/Internal/MutableColumn.hs diff --git a/dataframe-csv/src/DataFrame/IO/CSV.hs b/dataframe-csv/src/DataFrame/IO/CSV.hs index 7d8fa6d7..20a342af 100644 --- a/dataframe-csv/src/DataFrame/IO/CSV.hs +++ b/dataframe-csv/src/DataFrame/IO/CSV.hs @@ -1,11 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | CSV reading and writing for dataframes. The reader is a strict, -single-pass RFC 4180 scanner (cassava-compatible semantics, pinned by -the golden suite in @tests/IO/CsvGolden.hs@) that parses fields straight -into typed column builders. Ragged rows are padded with nulls (short -rows) and extra fields dropped. +{- | CSV reading and writing for dataframes. A strict, single-pass +RFC 4180 scanner (cassava-compatible) parses fields into typed column +builders; ragged rows are padded with nulls and extra fields dropped. -} module DataFrame.IO.CSV ( -- * Reading @@ -43,7 +41,7 @@ import Data.Maybe (fromMaybe) import DataFrame.IO.CSV.Internal.Options import DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) import DataFrame.Internal.DataFrame (DataFrame (..), toSeparated) -import DataFrame.Internal.Schema (Schema, elements) +import DataFrame.Schema (Schema, elements) {- | Read CSV file from path and load it into a dataframe. @@ -58,10 +56,8 @@ readCsv = readSeparated defaultReadOptions type CsvReader = Schema -> FilePath -> IO DataFrame -{- | Schema-driven CSV reader. Coerces each column to the type declared -in 'Schema'; columns absent from the schema fall back to the default -inference path. Defined in terms of 'readSeparated' with the 'TypeSpec' -filled in. +{- | Schema-driven CSV reader. Coerces each column to the type declared +in 'Schema'; columns absent from the schema fall back to inference. @ import qualified DataFrame as D diff --git a/dataframe-csv/src/DataFrame/IO/CSV/Internal/Options.hs b/dataframe-csv/src/DataFrame/IO/CSV/Internal/Options.hs index 1a359d56..db4b7b5f 100644 --- a/dataframe-csv/src/DataFrame/IO/CSV/Internal/Options.hs +++ b/dataframe-csv/src/DataFrame/IO/CSV/Internal/Options.hs @@ -19,8 +19,8 @@ module DataFrame.IO.CSV.Internal.Options ( import qualified Data.Map.Strict as M import qualified Data.Text as T -import DataFrame.Internal.Schema (SchemaType) import DataFrame.Operations.Typing (SafeReadMode (..)) +import DataFrame.Schema (SchemaType) data HeaderSpec = NoHeader | UseFirstRow | ProvideNames [T.Text] deriving (Eq, Show) diff --git a/dataframe-csv/src/DataFrame/IO/CSV/Internal/Read.hs b/dataframe-csv/src/DataFrame/IO/CSV/Internal/Read.hs index eb918b31..df29945a 100644 --- a/dataframe-csv/src/DataFrame/IO/CSV/Internal/Read.hs +++ b/dataframe-csv/src/DataFrame/IO/CSV/Internal/Read.hs @@ -4,10 +4,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Driver for the default CSV reader (Round-2 WS-D): a single strict -scan feeding per-column sinks, with cassava-parity semantics pinned by -@IO.CsvGolden@. Ragged rows pad trailing columns with null and drop -extra fields (audit D6). The result is fully forced (audit D7). +{- | Driver for the default CSV reader: a single strict scan feeding +per-column sinks, with cassava-parity semantics. Ragged rows pad +trailing columns with null, drop extra fields, and force the result. -} module DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) where @@ -32,12 +31,12 @@ import DataFrame.IO.CSV.Internal.Sink import DataFrame.Internal.Column (Column, ensureOptional) import DataFrame.Internal.ColumnBuilder import DataFrame.Internal.DataFrame (DataFrame (..), forceDataFrame) -import DataFrame.Internal.Schema (SchemaType (..), schemaType) import DataFrame.Operations.Typing ( SafeReadMode (..), effectiveSafeRead, parseWithTypes, ) +import DataFrame.Schema (SchemaType (..), schemaType) import Foreign.Ptr (castPtr) import Type.Reflection (typeRep) @@ -47,8 +46,6 @@ decodeCsvStrict opts bs = BSU.unsafeUseAsCStringLen bs $ \(cstr, _) -> do let !len = BS.length bs !sep = fromIntegral (ord (columnSeparator opts)) :: Word8 - -- Position of the next non-blank record at/after @pos@ (cassava - -- drops single-empty-field records); @len@ when none remain. findRecord !pos | pos >= len = len | otherwise = withField bs len sep pos $ \cs ce _ term next -> @@ -56,8 +53,6 @@ decodeCsvStrict opts bs = BSU.unsafeUseAsCStringLen bs $ \(cstr, _) -> do then if term == termEof then len else findRecord next else pos - -- Decode the record at @pos@ as header fields; returns the - -- position after the record. headerFields !pos = go pos id where go !p acc = withField bs len sep p $ \cs ce unesc term next -> @@ -82,8 +77,6 @@ decodeCsvStrict opts bs = BSU.unsafeUseAsCStringLen bs $ \(cstr, _) -> do when (dataPos >= len) (error "Empty CSV file") let resolveMode = effectiveSafeRead (safeRead opts) (safeReadOverrides opts) - -- If ANY column is EitherRead we keep every raw cell (including - -- "N/A" etc.) verbatim; otherwise the missing-indicator list applies. anyEither = any (\n -> resolveMode n == EitherRead) names missing = if anyEither then [] else missingIndicators opts missMode diff --git a/dataframe-expr-serializer/dataframe-expr-serializer.cabal b/dataframe-expr-serializer/dataframe-expr-serializer.cabal index 5d856cda..03aaf36f 100644 --- a/dataframe-expr-serializer/dataframe-expr-serializer.cabal +++ b/dataframe-expr-serializer/dataframe-expr-serializer.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-expr-serializer -version: 1.0.0.0 +version: 1.1.0.0 synopsis: Serialize and deserialize dataframe expressions and pipelines. description: @@ -40,8 +40,8 @@ library build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.13, - dataframe-core >= 1.1.1 && < 1.2, - dataframe-operations >= 1.1.2 && < 1.2, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-operations >= 2.0 && < 2.1, text >= 2.1 && < 3, vector ^>= 0.13 hs-source-dirs: src diff --git a/dataframe-fastcsv/dataframe-fastcsv.cabal b/dataframe-fastcsv/dataframe-fastcsv.cabal index 961d4e9c..34d0562b 100644 --- a/dataframe-fastcsv/dataframe-fastcsv.cabal +++ b/dataframe-fastcsv/dataframe-fastcsv.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-fastcsv -version: 1.1.1.1 +version: 1.2.0.0 synopsis: SIMD-accelerated CSV reader for the dataframe library. description: A fast, SIMD-accelerated CSV/TSV reader using memory-mapped I/O @@ -57,10 +57,11 @@ library build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9, - dataframe-core ^>= 1.1, - dataframe-csv ^>= 1.0.2, - dataframe-operations ^>= 1.1.1, - dataframe-parsing ^>= 1.0.2, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-csv >= 2.0 && < 2.1, + dataframe-operations >= 2.0 && < 2.1, + dataframe-parsing >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, mmap >= 0.5.8 && < 0.6, text >= 2.1 && < 3, time >= 1.12 && < 2, @@ -89,11 +90,12 @@ test-suite tests Properties.Csv build-depends: base >= 4 && < 5, containers >= 0.6.7 && < 0.9, - dataframe-core ^>= 1.1, - dataframe-csv ^>= 1.0.2, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-csv >= 2.0 && < 2.1, dataframe-fastcsv, - dataframe-operations ^>= 1.1.1, - dataframe-parsing ^>= 1.0.2, + dataframe-operations >= 2.0 && < 2.1, + dataframe-parsing >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, directory >= 1.3.0.0 && < 2, HUnit ^>= 1.6, QuickCheck >= 2 && < 3, diff --git a/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast.hs b/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast.hs index 30135a36..a4b58490 100644 --- a/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast.hs +++ b/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast.hs @@ -1,13 +1,6 @@ {- | SIMD-accelerated CSV reader: a C scanner finds delimiter positions, then each column is parsed directly from the mmap'd byte slices into -unboxed column builders (Round-2 typed extraction). Reads are strict: the -returned 'DataFrame' is fully built, with no deferred parse work. - -Extraction is chunk-parallel: inputs of 4 MB and up are split into one -row chunk per capability and parsed by 'Control.Concurrent.forkIO' -workers. The library never chooses RTS settings — to actually get -parallel reads, link the executable with @-threaded@ and run with -@+RTS -N@ (otherwise the reader falls back to the sequential path). +typed column builders. Chunk-parallel with @-threaded@ + @+RTS -N@. -} module DataFrame.IO.CSV.Fast ( fastReadCsv, @@ -50,7 +43,7 @@ import DataFrame.IO.CSV.Fast.Index ( tab, ) import DataFrame.Internal.DataFrame (DataFrame (..)) -import DataFrame.Internal.Schema (Schema (..)) +import DataFrame.Schema (Schema (..)) readSeparatedDefault :: Word8 -> FilePath -> IO DataFrame readSeparatedDefault separator = @@ -80,10 +73,8 @@ fastReadTsvWithOpts :: ReadOptions -> FilePath -> IO DataFrame fastReadTsvWithOpts = readSeparated tab {- | Read a CSV and coerce each column to the type declared in the -supplied 'Schema'. Schema columns bypass inference entirely: they are -parsed straight from the file bytes as the declared type, which is both -faster than inference and guards against the row-1 \"looks like Int\" -misclassification trap. +supplied 'Schema'. Schema columns bypass inference: parsed straight from +file bytes as the declared type, avoiding row-1 misclassification. -} fastReadCsvWithSchema :: Schema -> FilePath -> IO DataFrame fastReadCsvWithSchema schema = @@ -117,9 +108,8 @@ fastReadCsvProj projection path = do pure (projectColumns projection df) {- | Filter a DataFrame's columns down to (and in the order of) the -supplied names. Missing names are silently dropped rather than -raised; callers that want strict semantics can check the resulting -'columnIndices' map. +supplied names. Missing names are silently dropped; check the result's +'columnIndices' map for strict semantics. -} projectColumns :: [Text] -> DataFrame -> DataFrame projectColumns names df = @@ -129,14 +119,13 @@ projectColumns names df = (rows, _) = dataframeDimensions df in DataFrame newCols newIndices (rows, length idxs) M.empty +-- | Reads via a private ('WriteCopy') mmap so the buffer is never copied. readSeparated :: Word8 -> ReadOptions -> FilePath -> IO DataFrame readSeparated separator opts filePath = do - -- WriteCopy keeps the mapping private; the scanner never needs to - -- grow or pad the buffer, so the file is no longer copied (audit F6). (bufferPtr, offset, len) <- mmapFileForeignPtr filePath diff --git a/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast/Columns.hs b/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast/Columns.hs index 0d3ec037..ebf4bf0b 100644 --- a/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast/Columns.hs +++ b/dataframe-fastcsv/src/DataFrame/IO/CSV/Fast/Columns.hs @@ -4,10 +4,8 @@ {-# LANGUAGE TypeApplications #-} {- | Column-level dispatch for the fast CSV reader: resolve each column's -target type up front into a 'ColumnPlan' (schema hit -> typed pass directly; -miss -> sample classification + typed fallback chain; EitherRead \/ exotic -schema types -> the legacy Text pipeline). Plans are chunk-runnable so the -parallel reader (WS-E2) can execute them over row sub-ranges. +target type up front into a 'ColumnPlan' (schema hit, inference chain, or +legacy Text pipeline). Plans are chunk-runnable for the parallel reader. -} module DataFrame.IO.CSV.Fast.Columns ( ColumnEnv (..), @@ -46,7 +44,6 @@ import DataFrame.IO.CSV ( import DataFrame.IO.CSV.Fast.Passes import DataFrame.IO.CSV.Fast.Slice (extractField) import DataFrame.Internal.Column (Column, ensureOptional, fromVector) -import DataFrame.Internal.Schema (SchemaType (..)) import DataFrame.Operations.Typing ( ParseOptions (..), ParsingAssumption (..), @@ -56,6 +53,7 @@ import DataFrame.Operations.Typing ( makeParsingAssumption, parseFromExamples, ) +import DataFrame.Schema (SchemaType (..)) -- | One step of a type-fallback chain (always ends in 'StepText'). data Step = StepBool | StepInt | StepDouble | StepDate | StepText @@ -212,8 +210,7 @@ finishMode _ = id {- | Build one column sequentially (single chunk over the full row range). Returns the column plus a flag telling the caller to re-apply the legacy -schema conversion ('parseWithTypes') afterwards — only set for schema'd -columns the typed passes cannot serve. +'parseWithTypes' conversion afterwards (schema types the passes can't serve). -} buildColumn :: ColumnEnv -> T.Text -> Int -> IO (Column, Bool) buildColumn env name col = case planColumn env name col of diff --git a/dataframe-fastcsv/tests/Operations/ChunkParallel.hs b/dataframe-fastcsv/tests/Operations/ChunkParallel.hs index 6ceb41f8..786b1fd5 100644 --- a/dataframe-fastcsv/tests/Operations/ChunkParallel.hs +++ b/dataframe-fastcsv/tests/Operations/ChunkParallel.hs @@ -1,9 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | Round-2 (WS-E2) behavior pins for chunk-parallel extraction: a chunked -read must be indistinguishable from the sequential one — same column types -(promotion resolved across chunks), same values, same errors. +{- | Behavior pins for chunk-parallel extraction: a chunked read must be +indistinguishable from the sequential one — same column types (promotion +resolved across chunks), same values, and same errors. -} module Operations.ChunkParallel (tests) where @@ -23,8 +23,8 @@ import DataFrame.IO.CSV ( import qualified DataFrame.IO.CSV.Fast as D import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame, getColumn) -import DataFrame.Internal.Schema (Schema (..), SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..)) +import DataFrame.Schema (Schema (..), SchemaType (..), elements) import Test.HUnit comma :: Word8 @@ -205,8 +205,6 @@ testRaggedRowsAcrossChunks = TestLabel "par_ragged" . TestCase $ do -- Date column with nulls: boxed chunk columns must splice correctly. testDateNullsAcrossChunks :: Test testDateNullsAcrossChunks = TestLabel "par_date_nulls" . TestCase $ do - -- A second column keeps null rows from looking like blank lines - -- (blank lines are skipped by documented behavior). let cell k = if k `mod` 17 == 9 then "" @@ -240,9 +238,8 @@ testUnclosedQuoteAcrossChunks = TestLabel "par_unclosed_quote" . TestCase $ do Left D.CsvUnclosedQuote -> pure () other -> assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- WS-E2 ingest pin: the parallel byte-level merge wraps the merged shared --- buffer as 'PackedText' (no Text spine), so the parallel path is also an --- RSS win. Values stay identical to the sequential read. +-- The parallel byte-level merge wraps the merged buffer as 'PackedText' +-- (no Text spine), an RSS win; values stay identical to the sequential read. testParallelTextFreezesPacked :: Test testParallelTextFreezesPacked = TestLabel "par_text_freezes_packed" . TestCase $ do let body = diff --git a/dataframe-fastcsv/tests/Operations/ReadCsv.hs b/dataframe-fastcsv/tests/Operations/ReadCsv.hs index 910f0831..e21b7fe0 100644 --- a/dataframe-fastcsv/tests/Operations/ReadCsv.hs +++ b/dataframe-fastcsv/tests/Operations/ReadCsv.hs @@ -35,7 +35,7 @@ import DataFrame.Internal.DataFrame ( dataframeDimensions, getColumn, ) -import DataFrame.Internal.Schema (Schema (..), SchemaType (..)) +import DataFrame.Schema (Schema (..), SchemaType (..)) import System.Directory (removeFile) import System.IO (IOMode (..), withFile) import Test.HUnit @@ -64,7 +64,6 @@ prettyPrintSeparated sep filepath df = withFile filepath WriteMode $ \handle -> TIO.hPutStrLn handle (T.intercalate (T.singleton sep) (map (escapeField sep) headers)) - -- Write data rows mapM_ (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df) [0 .. rows - 1] @@ -193,7 +192,6 @@ testHeaderOnly = TestLabel "malformed_header_only" $ TestCase $ do testTrailingBlankLine :: Test testTrailingBlankLine = TestLabel "malformed_trailing_blank_line" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "trailing_blank_line.csv") - -- blank line contributes 1 extra delimiter; (3+3+1) div 3 = 2, numRow=1 assertEqual "trailing_blank_line.csv: 1 data row visible" 1 @@ -252,11 +250,9 @@ testWhitespaceFields = TestLabel "malformed_whitespace_fields" $ TestCase $ do (DI.fromList @T.Text [" New York", " Los Angeles"]) col --- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). --- After the row-index refactor each line is classified individually: --- the short row survives with a null-padded column 'c'; the full row --- keeps its value. This replaces the earlier "X bleeds from next row" --- behaviour, which was data corruption masquerading as a passing test. +-- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). Each line is +-- classified individually: the short row survives with a null-padded 'c'; +-- the full row keeps its value. testMissingFields :: Test testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "missing_fields.csv") @@ -279,10 +275,9 @@ testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do (DI.fromList @(Maybe T.Text) [Nothing, Just "Z"]) col --- File: a,b,c header; row "1,2,3,EXTRA" (over-long). --- Under the default ragged-row policy we read columns 0..numCol-1 and --- silently drop the EXTRA field. A stricter `RaggedRowPolicy = Error` --- will come with Step 7's ReadOptions knob. +-- File: a,b,c header; row "1,2,3,EXTRA" (over-long). Under the default +-- ragged-row policy we read columns 0..numCol-1 and silently drop the +-- EXTRA field. testExtraFieldsTruncate :: Test testExtraFieldsTruncate = TestLabel "malformed_extra_fields_truncate" $ TestCase $ do @@ -437,10 +432,9 @@ testProjection = TestLabel "projection" $ TestCase $ do map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df assertEqual "projection preserves order" ["c", "a"] names --- An unmatched `"` used to silently swallow the rest of the file: the --- SIMD scanner's PCLMUL quote-parity chain never resets, so every byte --- after the stray quote becomes "inside quotes." We now raise --- 'CsvUnclosedQuote' rather than returning a corrupted DataFrame. +-- An unmatched `"` used to silently swallow the rest of the file (the SIMD +-- quote-parity chain never resets). We now raise 'CsvUnclosedQuote' rather +-- than returning a corrupted DataFrame. testUnclosedQuote :: Test testUnclosedQuote = TestLabel "malformed_unclosed_quote" $ TestCase $ do let path = fixtureDir <> "unclosed_quote.csv" diff --git a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs index 764afc99..c99bd534 100644 --- a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs +++ b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs @@ -2,9 +2,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Round-2 (WS-E1) behavior pins for the typed-extraction fastcsv path: -schema bypass, unified WS-B parser semantics, byte-level missing tests, -and the chunk-boundary scalar tail that replaced the full-file copy. +{- | Behavior pins for the typed-extraction fastcsv path: schema bypass, +parser semantics, byte-level missing tests, and the chunk-boundary scalar +tail that replaced the full-file copy. -} module Operations.TypedExtraction (tests) where @@ -25,8 +25,8 @@ import qualified DataFrame.IO.CSV.Fast as D import DataFrame.Internal.Column (Column) import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame, dataframeDimensions, getColumn) -import DataFrame.Internal.Schema (Schema (..), SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..)) +import DataFrame.Schema (Schema (..), SchemaType (..), elements) import System.Directory (removeFile) import Test.HUnit @@ -59,10 +59,9 @@ testIntOverflowDemotesToDouble = TestLabel "typed_int_overflow_demotes" $ df (DI.fromList @Double [1.0, 9.223372036854776e18]) --- Divergence #2 (pinned new behavior): doubles parse with strip-equivalent --- semantics, so a padded double past the 100-row inference sample no longer --- demotes the whole column to Text. (Inside the sample the Text-level --- classifier still rejects padding, by design.) +-- Divergence #2: doubles parse with strip-equivalent semantics, so a padded +-- double past the 100-row inference sample no longer demotes the column to +-- Text. (Inside the sample the classifier still rejects padding.) testPaddedDoubleStaysDouble :: Test testPaddedDoubleStaysDouble = TestLabel "typed_padded_double" $ TestCase $ do @@ -215,9 +214,8 @@ testDateWithNulls = TestLabel "typed_date_nulls" $ ) -- The scalar tail (last <64 bytes after the SIMD chunks) must honour the --- quote state carried out of the SIMD region: a quoted field that opens --- before the 64-byte boundary and closes inside the tail may contain --- separators and newlines. +-- quote state from the SIMD region: a quoted field opening before the +-- boundary and closing in the tail may contain separators and newlines. testQuoteSpansChunkBoundary :: Test testQuoteSpansChunkBoundary = TestLabel "typed_quote_spans_boundary" $ TestCase $ do diff --git a/dataframe-fusion/dataframe-fusion.cabal b/dataframe-fusion/dataframe-fusion.cabal index 0c748798..4fa1aa58 100644 --- a/dataframe-fusion/dataframe-fusion.cabal +++ b/dataframe-fusion/dataframe-fusion.cabal @@ -68,10 +68,11 @@ library build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.13, aeson >= 0.11 && < 3, - dataframe-core ^>= 1.1, - dataframe-lazy ^>= 1.1, - dataframe-operations ^>= 1.1.1, - dataframe:arrow-bridge >= 1 && < 3, + dataframe-core ^>= 2.0, + dataframe-core:internal ^>= 2.0, + dataframe-lazy ^>= 2.0, + dataframe-operations ^>= 2.0, + dataframe:arrow-bridge >= 1 && < 4, text >= 2.1 && < 3 hs-source-dirs: src include-dirs: cbits @@ -91,10 +92,10 @@ test-suite tests if flag(no-csv) || flag(no-parquet) buildable: False build-depends: base >= 4 && < 5, - dataframe >= 1 && < 3, - dataframe-core ^>= 1.1, + dataframe >= 1 && < 4, + dataframe-core ^>= 2.0, dataframe-fusion, - dataframe-operations ^>= 1.1.1, + dataframe-operations ^>= 2.0, directory >= 1.3 && < 2, filepath >= 1.4 && < 2, HUnit ^>= 1.6, diff --git a/dataframe-fusion/src/DataFrame/Fusion/Typed.hs b/dataframe-fusion/src/DataFrame/Fusion/Typed.hs index 35cfac1c..a8a0ed8a 100644 --- a/dataframe-fusion/src/DataFrame/Fusion/Typed.hs +++ b/dataframe-fusion/src/DataFrame/Fusion/Typed.hs @@ -7,7 +7,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} --- | Typed DataFusion-backed query API. Mirrors 'DataFrame.Typed.Lazy' with: +{- | Typed DataFusion-backed query API. Mirrors 'DataFrame.Typed.Lazy' but runs +on Rust-side DataFusion via FFI. +-} module DataFrame.Fusion.Typed ( -- * Carrier DataFrame, @@ -59,7 +61,7 @@ import Prelude hiding (filter, take) import qualified DataFrame.Internal.Column as IC import qualified DataFrame.Internal.Expression as IE -import DataFrame.Lazy.Internal.LogicalPlan (SortOrder (..)) +import DataFrame.Lazy (SortOrder (..)) import qualified DataFrame.Typed.Aggregate as AGG import qualified DataFrame.Fusion.FFI as F @@ -76,11 +78,9 @@ underlying handle is owned by Rust-side DataFusion; Haskell holds a -} newtype DataFrame (cols :: [Type]) = DF {unDataFrame :: PlanHandle} -{- | Scan a CSV file. The schema is taken from the @cols@ phantom via -'KnownSchema'; the user does not pass it separately. DataFusion currently -infers column types from the file, so the constraint is not used for -type override yet — it is in place so v1.5 can derive the wire-format -schema JSON from the type-level @cols@ list and pass it to Rust. +{- | Scan a CSV file. The schema comes from the @cols@ phantom via 'KnownSchema'; +the user does not pass it separately. DataFusion currently infers column types +from the file, so the constraint is reserved for future wire-format derivation. -} scanCsv :: forall cols. @@ -131,10 +131,9 @@ select (DF plan) = do ph <- runPlanOp (F.df_plan_select pp cjson) return (DF ph) -{- | Add a computed column. Result schema is the input schema with the new -column appended (mirroring 'DataFrame.Typed.Lazy.derive'). The expression -is lowered to JSON via 'encodeExprToBytes' and decoded into a DataFusion -'Expr' on the Rust side. +{- | Add a computed column, appended to the input schema (mirroring +'DataFrame.Typed.Lazy.derive'). The expression is lowered to JSON and decoded +into a DataFusion 'Expr' on the Rust side. -} derive :: forall name a cols. @@ -168,9 +167,8 @@ groupBy :: groupBy (DF plan) = GD (symbolVals @keys) plan {- | Aggregate a grouped query. The first argument is a chain of 'AGG.as' -entries composed with @(.)@; the empty composition (@id@) yields just -the group keys. The wire format mirrors the JSON shape decoded by -@df_plan_groupby_aggregate@ on the Rust side. +entries composed with @(.)@; the empty composition (@id@) yields just the +group keys. -} aggregate :: forall keys cols aggs. diff --git a/dataframe-hasktorch/dataframe-hasktorch.cabal b/dataframe-hasktorch/dataframe-hasktorch.cabal index 3338f89d..1ea5f533 100644 --- a/dataframe-hasktorch/dataframe-hasktorch.cabal +++ b/dataframe-hasktorch/dataframe-hasktorch.cabal @@ -37,8 +37,8 @@ library build-depends: base >= 4.11 && < 5, vector ^>= 0.13, - dataframe-core ^>= 1.1, - dataframe-operations ^>= 1.1.1, + dataframe-core ^>= 2.0, + dataframe-operations ^>= 2.0, hasktorch >= 0.2.1.6 && < 0.3 if impl(ghc >= 9.12) build-depends: ghc-typelits-natnormalise == 0.9.3 diff --git a/dataframe-hasktorch/src/DataFrame/Hasktorch.hs b/dataframe-hasktorch/src/DataFrame/Hasktorch.hs index 33168033..2375c140 100644 --- a/dataframe-hasktorch/src/DataFrame/Hasktorch.hs +++ b/dataframe-hasktorch/src/DataFrame/Hasktorch.hs @@ -11,41 +11,15 @@ import qualified Data.Vector.Unboxed.Mutable as VUM import qualified DataFrame.Operations.Core as D import Control.Exception (throw) -import DataFrame.Internal.DataFrame (DataFrame) +import DataFrame.Core (DataFrame) import Torch -{- | Converts a dataframe to a floating-point tensor. - -This function converts all columns in the dataframe to floats and creates -a tensor suitable for machine learning operations. The tensor dimensions -are determined by the dataframe's shape. - -==== __Dimensional behavior__ - -* Multi-column dataframe: Creates a 2D tensor with shape @[rows, columns]@ -* Single-column dataframe: Creates a 1D tensor with shape @[rows]@ - -==== __Conversion process__ - -1. Converts the dataframe to a float matrix using 'D.toFloatMatrix' -2. Flattens the matrix features into a 1D representation -3. Reshapes into the appropriate tensor dimensions - -==== __Throws__ - -* 'DataFrameException' - if any column cannot be converted to float - -==== __Examples__ +{- | Convert a dataframe to a floating-point tensor of shape @[rows, columns]@ +(or @[rows]@ when single-column). Throws 'DataFrameException' if a column +cannot be converted to float. >>> toTensor df -- where df has shape (100, 5) Tensor with shape [100, 5] - ->>> toTensor df -- where df has shape (100, 1) -Tensor with shape [100] - -==== __See also__ - -* 'toIntTensor' - for integer tensor conversion -} toTensor :: DataFrame -> Tensor toTensor df = case D.toFloatMatrix df of @@ -57,43 +31,12 @@ toTensor df = case D.toFloatMatrix df of in reshape dims' (asTensor (flattenFeatures m)) -{- | Converts a dataframe to an integer tensor. - -This function converts all columns in the dataframe to integers and creates -a tensor suitable for machine learning operations (e.g., classification labels, -discrete features). The tensor dimensions are determined by the dataframe's shape. - -==== __Dimensional behavior__ - -* Multi-column dataframe: Creates a 2D tensor with shape @[rows, columns]@ -* Single-column dataframe: Creates a 1D tensor with shape @[rows]@ - -==== __Conversion process__ - -1. Converts the dataframe to an int matrix using 'D.toIntMatrix' -2. Flattens the matrix features into a 1D representation -3. Reshapes into the appropriate tensor dimensions - -==== __Throws__ - -* 'DataFrameException' - if any column cannot be converted to int - -==== __Examples__ - ->>> toIntTensor labelsDf -- where labelsDf has shape (100, 1) -Tensor with shape [100] +{- | Convert a dataframe to an integer tensor of shape @[rows, columns]@ (or +@[rows]@ when single-column). Floating-point values are rounded. Throws +'DataFrameException' if a column cannot be converted to int. >>> toIntTensor featuresDf -- where featuresDf has shape (100, 3) Tensor with shape [100, 3] - -==== __Note__ - -Floating-point values in the dataframe will be rounded to the nearest integer. -See 'D.toIntMatrix' for details on the conversion behavior. - -==== __See also__ - -* 'toTensor' - for floating-point tensor conversion -} toIntTensor :: DataFrame -> Tensor toIntTensor df = case D.toIntMatrix df of diff --git a/dataframe-huggingface/dataframe-huggingface.cabal b/dataframe-huggingface/dataframe-huggingface.cabal index 850e99aa..fcbd6818 100644 --- a/dataframe-huggingface/dataframe-huggingface.cabal +++ b/dataframe-huggingface/dataframe-huggingface.cabal @@ -33,11 +33,11 @@ library build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.13, - dataframe-core ^>= 1.1, - dataframe-lazy ^>= 1.1, - dataframe-operations ^>= 1.1.1, - dataframe-parquet ^>= 1.1, - dataframe-parsing ^>= 1.0.2, + dataframe-core ^>= 2.0, + dataframe-lazy ^>= 2.0, + dataframe-operations ^>= 2.0, + dataframe-parquet ^>= 1.2, + dataframe-parsing ^>= 2.0, directory >= 1.3.0.0 && < 2, filepath >= 1.4 && < 2, Glob >= 0.10 && < 1, diff --git a/dataframe-huggingface/src/DataFrame/IO/HuggingFace.hs b/dataframe-huggingface/src/DataFrame/IO/HuggingFace.hs index 0ebc2423..c6f32425 100644 --- a/dataframe-huggingface/src/DataFrame/IO/HuggingFace.hs +++ b/dataframe-huggingface/src/DataFrame/IO/HuggingFace.hs @@ -1,16 +1,10 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | Read Parquet datasets from HuggingFace (@hf://@) into 'DataFrame's. - -A @hf://@ URI has the form @hf:\/\/datasets\/{owner}\/{dataset}\/{glob}@. -When the glob contains wildcards the dataset's file list is resolved via the -HuggingFace datasets-server API; otherwise the file is fetched directly from -the repo. Resolved files are downloaded to a temporary location and then read -with the local "DataFrame.IO.Parquet" reader. - -This module is the home for the heavier @aeson@ and @http-conduit@ -dependencies, keeping @dataframe-parquet@ and @dataframe-lazy@ free of them. +{- | Read Parquet datasets from HuggingFace URIs +(@hf:\/\/datasets\/{owner}\/{dataset}\/{glob}@) into 'DataFrame's. Globs are +resolved via the datasets-server API, files downloaded to a temp dir, then read +with the local "DataFrame.IO.Parquet". -} module DataFrame.IO.HuggingFace ( -- * Eager readers @@ -49,16 +43,16 @@ import qualified Data.ByteString as BS import qualified Data.List as L import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8) +import DataFrame.Core (DataFrame) import DataFrame.IO.Parquet ( ParquetReadOptions (..), defaultParquetReadOptions, ) import qualified DataFrame.IO.Parquet as Parquet -import DataFrame.Internal.DataFrame (DataFrame) -import DataFrame.Internal.Schema (Schema) import qualified DataFrame.Lazy as Lazy import DataFrame.Operations.Merge () import qualified DataFrame.Operations.Subset as DS +import DataFrame.Schema (Schema) import Network.HTTP.Simple ( getResponseBody, getResponseStatusCode, @@ -121,10 +115,8 @@ applyRowRange opts df = -- Lazy / streaming reader ------------------------------------------------- {- | Build a lazy 'Lazy.LazyDataFrame' over a @hf://@ dataset (or local path). - -The HuggingFace files are resolved and downloaded to a temporary directory up -front, then scanned lazily from local disk — so the query runs in constant -memory but the whole dataset is fetched before scanning begins. +HuggingFace files are downloaded to a temp dir up front, then scanned lazily +from disk — constant-memory query, but the whole dataset is fetched first. -} scanParquet :: Schema -> T.Text -> IO Lazy.LazyDataFrame scanParquet schema uri @@ -207,7 +199,6 @@ hfUrlRepoPath f = case T.breakOn "/resolve/" (hfpUrl f) of (_, rest) | not (T.null rest) -> - -- Drop "/resolve/", then drop the ref component (up to and including "/") T.unpack $ T.drop 1 $ T.dropWhile (/= '/') $ T.drop (T.length "/resolve/") rest _ -> T.unpack (hfpConfig f) T.unpack (hfpSplit f) T.unpack (hfpFilename f) @@ -247,7 +238,6 @@ downloadHFFilesTo :: FilePath -> Maybe BS.ByteString -> [HFParquetFile] -> IO [FilePath] downloadHFFilesTo destDir mToken files = forM files $ \f -> do - -- Derive a collision-resistant name from the URL path components let fname = case (hfpConfig f, hfpSplit f) of (c, s) | T.null c && T.null s -> T.unpack (hfpFilename f) (c, s) -> T.unpack c <> "_" <> T.unpack s <> "_" <> T.unpack (hfpFilename f) diff --git a/dataframe-json/dataframe-json.cabal b/dataframe-json/dataframe-json.cabal index b214664b..b7acf59b 100644 --- a/dataframe-json/dataframe-json.cabal +++ b/dataframe-json/dataframe-json.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-json -version: 1.0.1.1 +version: 1.1.0.0 synopsis: JSON reader and writer for the dataframe ecosystem. description: @@ -30,7 +30,7 @@ library build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.13, - dataframe-core ^>= 1.1, + dataframe-core:internal >= 2.0 && < 2.1, scientific >= 0.3.1 && < 0.4, text >= 2.1 && < 3, vector ^>= 0.13 diff --git a/dataframe-lazy/dataframe-lazy.cabal b/dataframe-lazy/dataframe-lazy.cabal index 36aabaac..048804d2 100644 --- a/dataframe-lazy/dataframe-lazy.cabal +++ b/dataframe-lazy/dataframe-lazy.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-lazy -version: 1.1.0.2 +version: 2.0.0.0 synopsis: Lazy query engine for the dataframe ecosystem. description: The lazy/streaming query engine: relational-algebra plans, optimizer, @@ -31,22 +31,28 @@ library DataFrame.Lazy DataFrame.Lazy.IO.Binary DataFrame.Lazy.IO.CSV + DataFrame.Typed.Lazy + -- Relational-algebra plan tree, optimizer, and pull-based executor: + -- sealed. SortOrder(..)/LazyDataFrame re-exported via DataFrame.Lazy. + other-modules: DataFrame.Lazy.Internal.DataFrame DataFrame.Lazy.Internal.Executor DataFrame.Lazy.Internal.LogicalPlan DataFrame.Lazy.Internal.Optimizer DataFrame.Lazy.Internal.PhysicalPlan - DataFrame.Typed.Lazy build-depends: base >= 4 && < 5, async >= 2.2 && < 3, attoparsec >= 0.12 && < 0.15, bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9, - dataframe-core ^>= 1.1, - dataframe-csv ^>= 1.0.2, - dataframe-operations ^>= 1.1.1, - dataframe-parquet ^>= 1.1, - dataframe-parsing ^>= 1.0.2, + dataframe-core >= 2.0 && < 2.1, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-csv >= 2.0 && < 2.1, + dataframe-csv:internal >= 2.0 && < 2.1, + dataframe-operations >= 2.0 && < 2.1, + dataframe-parquet >= 1.2 && < 1.3, + dataframe-parsing >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, directory >= 1.3.0.0 && < 2, filepath >= 1.4 && < 2, Glob >= 0.10 && < 1, diff --git a/dataframe-lazy/src/DataFrame/Lazy.hs b/dataframe-lazy/src/DataFrame/Lazy.hs index 9d69f7f2..1ce652c8 100644 --- a/dataframe-lazy/src/DataFrame/Lazy.hs +++ b/dataframe-lazy/src/DataFrame/Lazy.hs @@ -1,3 +1,7 @@ -module DataFrame.Lazy (module DataFrame.Lazy.Internal.DataFrame) where +module DataFrame.Lazy ( + module DataFrame.Lazy.Internal.DataFrame, + SortOrder (..), +) where import DataFrame.Lazy.Internal.DataFrame +import DataFrame.Lazy.Internal.LogicalPlan (SortOrder (..)) diff --git a/dataframe-lazy/src/DataFrame/Lazy/IO/CSV.hs b/dataframe-lazy/src/DataFrame/Lazy/IO/CSV.hs index fad4dea2..c3a24c15 100644 --- a/dataframe-lazy/src/DataFrame/Lazy/IO/CSV.hs +++ b/dataframe-lazy/src/DataFrame/Lazy/IO/CSV.hs @@ -36,8 +36,8 @@ import DataFrame.Internal.Column ( ) import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing -import DataFrame.Internal.Schema (Schema, SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..), effectiveSafeRead) +import DataFrame.Schema (Schema, SchemaType (..), elements) import System.IO import Type.Reflection import Prelude hiding (takeWhile) @@ -59,11 +59,9 @@ data ReadOptions = ReadOptions , rowsRead :: !Int } -{- | By default we assume the file has a header and we infer types on read. -'safeRead' starts as 'NoSafeRead' — set it to 'MaybeRead' to wrap columns as -@Maybe a@, or 'EitherRead' to wrap as @Either Text a@ preserving the raw text -of any rows that fail to parse. Use 'safeReadOverrides' to pick a different -mode for specific columns. +{- | Default read options: assume a header, infer types, no safe-read wrapping. +Set 'safeRead' to 'MaybeRead'/'EitherRead' to wrap columns; use +'safeReadOverrides' to pick a different mode per column. -} defaultOptions :: ReadOptions defaultOptions = @@ -110,30 +108,24 @@ readSeparated c opts path = do if hasHeader opts then fmap (T.filter (/= '\"')) firstRow else fmap (T.singleton . intToDigit) [0 .. (length firstRow - 1)] - -- If there was no header rewind the file cursor. unless (hasHeader opts) $ hSeek handle AbsoluteSeek 0 currPos <- hTell handle when (isJust $ seekPos opts) $ hSeek handle AbsoluteSeek (fromMaybe currPos (seekPos opts)) - -- Initialize mutable vectors for each column let numColumns = length columnNames let numRows = len' - -- Use this row to infer the types of the rest of the column. (dataRow, remainder) <- readSingleLine c (leftOver opts) handle - -- This array will track the indices of all null values for each column. nullIndices <- VM.unsafeNew numColumns VM.set nullIndices [] mutableCols <- VM.unsafeNew numColumns getInitialDataVectors numRows mutableCols dataRow - -- Read rows into the mutable vectors (unconsumed, r) <- fillColumns numRows c mutableCols nullIndices remainder handle - -- Freeze the mutable vectors into immutable ones nulls' <- V.unsafeFreeze nullIndices let !columnNamesV = V.fromList columnNames cols <- @@ -252,11 +244,9 @@ freezeColumn colNames mutableCols nulls opts colIndex = do -- Streaming scan API -- --------------------------------------------------------------------------- -{- | Open a CSV/separated file for streaming, returning an open handle -(positioned just after the header line) and the column specification -for the schema columns that appear in the file header. - -The caller is responsible for closing the handle when done. +{- | Open a file for streaming: returns a handle positioned after the header +and the column spec for the schema columns present in the header. +The caller must close the handle when done. -} openCsvStream :: Char -> @@ -280,12 +270,9 @@ openCsvStream sep schema path = do ("openCsvStream: none of the schema columns appear in the header of " <> path) return (handle, colSpec) -{- | Read up to @batchSz@ rows from the open handle, returning a batch -'DataFrame' and the unconsumed leftover text. Returns 'Nothing' when -the handle is at EOF and there is no leftover input. - -The caller must pass the leftover returned by the previous call (use @""@ -for the first call). +{- | Read up to @batchSz@ rows, returning a batch 'DataFrame' and the unconsumed +leftover text; 'Nothing' at EOF with no leftover. Pass the previous call's +leftover back in (use @""@ on the first call). -} readBatch :: Char -> @@ -297,17 +284,13 @@ readBatch :: readBatch sep colSpec batchSz leftover handle = do let sepByte = fromIntegral (fromEnum sep) :: Word8 numCols = length colSpec - -- Read in 8 MB chunks; only the partial-line tail is copied on refill. chunkSize = 8 * 1024 * 1024 nullsArr <- VM.unsafeNew numCols VM.set nullsArr [] mCols <- VM.unsafeNew numCols forM_ (zip [0 ..] colSpec) $ \(ci, (_, _, st)) -> VM.unsafeWrite mCols ci =<< makeCol batchSz st - -- buf holds unprocessed bytes; refilled on demand when no newline is found. bufRef <- newIORef leftover - -- Row-by-row scan. When the buffer has no unquoted newline, fetch another chunk. - -- The copy on refill is only the partial-line tail (≤ one row ≈ few hundred bytes). let loop !rowIdx = do remaining <- readIORef bufRef if rowIdx >= batchSz @@ -316,7 +299,7 @@ readBatch sep colSpec batchSz leftover handle = do Nothing -> do chunk <- BS.hGet handle chunkSize if BS.null chunk - then return (rowIdx, remaining) -- EOF + then return (rowIdx, remaining) else writeIORef bufRef (remaining <> chunk) >> loop rowIdx Just nlIdx -> do let line = BS.take nlIdx remaining @@ -389,7 +372,6 @@ getNthFieldBs sep targetIdx bs | not (BS.any (== 0x22) bs) = skipFast targetIdx bs | otherwise = go 0 0 False 0 where - -- Fast path: skip fields using elemIndex (memchr); avoids pair allocation. skipFast k s = case BS.elemIndex sep s of Nothing -> if k == 0 then s else BS.empty @@ -398,7 +380,6 @@ getNthFieldBs sep targetIdx bs then BS.take i s else skipFast (k - 1) (BS.drop (i + 1) s) - -- Slow path: quote-aware scan. quoteChar = 0x22 :: Word8 len = BS.length bs go !idx !start !inQ !pos @@ -448,10 +429,7 @@ findUnquotedNewline bs = case BS.elemIndex 0x0A bs of Nothing -> Nothing Just nlPos - -- No quote before the newline → safe to use this position. - -- Check with elemIndex to avoid allocating a ByteString slice. | maybe True (>= nlPos) (BS.elemIndex 0x22 bs) -> Just nlPos - -- Quote present → may be a newline inside a quoted field; scan carefully. | otherwise -> slowScan 0 False where len = BS.length bs diff --git a/dataframe-lazy/src/DataFrame/Lazy/Internal/DataFrame.hs b/dataframe-lazy/src/DataFrame/Lazy/Internal/DataFrame.hs index bf0de369..1d99f048 100644 --- a/dataframe-lazy/src/DataFrame/Lazy/Internal/DataFrame.hs +++ b/dataframe-lazy/src/DataFrame/Lazy/Internal/DataFrame.hs @@ -12,7 +12,6 @@ import DataFrame.IO.CSV (CsvReader, readCsvWithSchema) import qualified DataFrame.Internal.Column as C import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.Expression as E -import DataFrame.Internal.Schema (Schema) import DataFrame.Lazy.Internal.Executor (execute) import DataFrame.Lazy.Internal.LogicalPlan ( DataSource (..), @@ -21,11 +20,10 @@ import DataFrame.Lazy.Internal.LogicalPlan ( ) import qualified DataFrame.Lazy.Internal.Optimizer as Opt import DataFrame.Operations.Join (JoinType) +import DataFrame.Schema (Schema) -{- | A lazy query that has not been executed yet. - -The query is represented as a 'LogicalPlan' tree; execution is deferred -until 'runDataFrame' is called. +{- | A lazy query that has not been executed yet: a 'LogicalPlan' tree whose +execution is deferred until 'runDataFrame' is called. -} data LazyDataFrame = LazyDataFrame { plan :: LogicalPlan @@ -42,9 +40,7 @@ instance Show LazyDataFrame where -- --------------------------------------------------------------------------- {- | Execute the lazy query: optimise the logical plan, then stream-execute -the resulting physical plan, returning a fully-materialised 'D.DataFrame'. -The CSV reader (default: attoparsec) is set per scan via 'scanCsv' / -'scanCsvWith'. +the resulting physical plan into a fully-materialised 'D.DataFrame'. -} runDataFrame :: LazyDataFrame -> IO D.DataFrame runDataFrame ldf = execute (Opt.optimize (batchSize ldf) (plan ldf)) diff --git a/dataframe-lazy/src/DataFrame/Lazy/Internal/Executor.hs b/dataframe-lazy/src/DataFrame/Lazy/Internal/Executor.hs index 90d0174b..9c381e8b 100644 --- a/dataframe-lazy/src/DataFrame/Lazy/Internal/Executor.hs +++ b/dataframe-lazy/src/DataFrame/Lazy/Internal/Executor.hs @@ -8,29 +8,9 @@ {-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} -{- | Pull-based (iterator) execution engine. - -Each operator returns a 'Stream' — an IO action that produces the next -'DataFrame' batch on each call and returns 'Nothing' when exhausted. -Blocking operators (Sort, HashJoin) materialise their input before producing -output. HashAggregate uses streaming partial aggregation when all aggregate -expressions support it. - -== Streaming vs. materialize routing - -When the source(s) of an operator are BOUNDED (finite local CSV/Parquet -files — the db-benchmark / join-pipeline case), 'HashJoin' and -'HashAggregate' materialise their input and call the whole-frame optimized -eager op (parallel 'Join.join' with salt + ParRadixSort + parallel probe; -'Agg.aggregate' . 'Agg.groupBy' with parallel partitioned + low-card-direct -grouping). This inherits the eager Rounds 5-10 gains and produces results -byte-identical to the eager path. - -When a source is UNBOUNDED (an online/streaming source), the per-batch -streaming partial-aggregation / streaming-probe paths are preserved so the -query runs in constant memory. Boundedness is read off the physical plan -leaves by 'isBounded'. All current scan leaves are local files, so the -streaming paths are a latent capability with no in-tree producer. +{- | Pull-based (iterator) execution engine: each operator returns a 'Stream' +yielding the next 'DataFrame' batch or 'Nothing' at end. Bounded local sources +materialise into eager whole-frame ops; unbounded sources stream in constant memory. -} module DataFrame.Lazy.Internal.Executor ( CsvReader, @@ -62,7 +42,6 @@ import qualified DataFrame.IO.Parquet as Parquet import qualified DataFrame.Internal.Column as C import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.Expression as E -import DataFrame.Internal.Schema (elements) import qualified DataFrame.Lazy.IO.Binary as Bin import DataFrame.Lazy.Internal.LogicalPlan (DataSource (..), SortOrder (..)) import DataFrame.Lazy.Internal.PhysicalPlan @@ -73,6 +52,7 @@ import DataFrame.Operations.Merge () import qualified DataFrame.Operations.Permutation as Perm import qualified DataFrame.Operations.Subset as Sub import qualified DataFrame.Operations.Transformations as Trans +import DataFrame.Schema (elements) import System.Directory (doesDirectoryExist, removeFile) import System.FilePath (()) import System.FilePath.Glob (glob) @@ -102,12 +82,8 @@ materialized df = do return mb {- | Drain all batches from a stream and concatenate them into one DataFrame. - -Batches are buffered, then each column is concatenated across all batches in a -single multi-way pass ('C.concatManyColumns', backed by 'VU.concat' / -'VB.concat'). A left-fold of @acc <> batch@ would copy the growing -accumulator on every step (≈ O(rows × batches)); this is O(rows) and keeps the -bounded materialize path on par with a single eager whole-frame read. +Columns are concatenated in a single multi-way pass (O(rows)) rather than a +left-fold of @acc <> batch@ that would recopy the accumulator on every step. -} collectStream :: Stream -> IO D.DataFrame collectStream stream = go [] @@ -132,14 +108,9 @@ concatBatches batches@(first : _) = -- Boundedness analysis -- --------------------------------------------------------------------------- -{- | A plan is BOUNDED when every scan leaf reads a finite local source -(local CSV or local Parquet files): the whole result can be materialised, so -blocking/bounded operators route through the whole-frame fast eager ops. - -A plan is UNBOUNDED when any leaf is an online/streaming source, in which case -the streaming partial-aggregation / streaming-probe paths are kept so the query -runs in constant memory. No in-tree scan produces an unbounded source today, -so this always holds; the routing is retained for future streaming sources. +{- | True when every scan leaf is a finite local source, so the whole result can +be materialised and routed through the eager whole-frame ops. False for +online/streaming sources, which keep the constant-memory streaming paths. -} isBounded :: PhysicalPlan -> Bool isBounded (PhysicalScan (CsvSource{}) _) = True @@ -187,7 +158,6 @@ foldBatches f seed plan = do -- --------------------------------------------------------------------------- buildStream :: PhysicalPlan -> IO Stream --- Scan ----------------------------------------------------------------------- buildStream (PhysicalScan (CsvSource path sep reader) cfg) = executeCsvScan path sep reader cfg buildStream (PhysicalScan (CsvSourceStreaming path _sep reader) cfg) = @@ -199,7 +169,6 @@ buildStream (PhysicalSpill child path) = do Bin.spillToDisk path df df' <- Bin.readSpilled path materialized df' --- Filter --------------------------------------------------------------------- buildStream (PhysicalFilter p child) = do childStream <- buildStream child return . Stream $ @@ -207,7 +176,6 @@ buildStream (PhysicalFilter p child) = do mb <- pullBatch childStream return $ fmap (Sub.filterWhere p) mb ) --- Project -------------------------------------------------------------------- buildStream (PhysicalProject cols child) = do childStream <- buildStream child return . Stream $ @@ -215,7 +183,6 @@ buildStream (PhysicalProject cols child) = do mb <- pullBatch childStream return $ fmap (Sub.select cols) mb ) --- Derive --------------------------------------------------------------------- buildStream (PhysicalDerive name uexpr child) = do childStream <- buildStream child return . Stream $ @@ -223,7 +190,6 @@ buildStream (PhysicalDerive name uexpr child) = do mb <- pullBatch childStream return $ fmap (Trans.deriveMany [(name, uexpr)]) mb ) --- Limit ---------------------------------------------------------------------- buildStream (PhysicalLimit n child) = do childStream <- buildStream child countRef <- newIORef (0 :: Int) @@ -241,16 +207,11 @@ buildStream (PhysicalLimit n child) = do modifyIORef' countRef (+ toTake) return $ Just (Sub.take toTake df) ) --- Sort (blocking) ------------------------------------------------------------ buildStream (PhysicalSort cols child) = do df <- execute child let sortOrds = fmap (toPermSortOrder df) cols materialized (Perm.sortBy sortOrds df) --- HashAggregate -------------------------------------------------------------- buildStream (PhysicalHashAggregate keys aggs child) - -- BOUNDED child: materialise then run the whole-frame parallel grouping - -- (parallel partitioned + low-card-direct + vectorized scatter). Same - -- result the eager path produces; small-batch partial-agg overhead avoided. | isBounded child = do df <- execute child let result = Agg.aggregate aggs (Agg.groupBy keys df) @@ -259,12 +220,6 @@ buildStream (PhysicalHashAggregate keys aggs child) = do childStream <- buildStream child if all (isStreamableAgg . snd) aggs then do - -- Parallel streaming partial aggregation: - -- * N workers, each pulls batches from the child stream and - -- maintains its own local accumulator. - -- * Once the stream is drained, the N partials are merged - -- sequentially using the same merge expression. - -- * O(|groups| × N) memory in flight, then O(|groups|). let (partialAggs, mergeAggs, finalizer) = buildAggPlan aggs nCaps <- getNumCapabilities let workers = max 1 nCaps @@ -286,10 +241,8 @@ buildStream (PhysicalHashAggregate keys aggs child) = do writeIORef ref Nothing return mb else do - -- Fallback: materialise entire child (for CollectAgg etc.) df <- collectStream childStream materialized (Agg.aggregate aggs (Agg.groupBy keys df)) --- SourceDF (split pre-loaded DataFrame into batches) ------------------------- buildStream (PhysicalSourceDF bs df) = do let total = Core.nRows df posRef <- newIORef (0 :: Int) @@ -302,12 +255,7 @@ buildStream (PhysicalSourceDF bs df) = do batch = Sub.range (i, i + n) df writeIORef posRef (i + n) return (Just batch) --- HashJoin — streaming probe (INNER/LEFT) or blocking fallback ---------------- buildStream (PhysicalHashJoin jt leftKey rightKey leftPlan rightPlan) - -- BOUNDED probe side: materialise both sides and call the whole-frame eager - -- join (parallel probe + ParRadixSort + salt). Inherits Rounds 5/8/10 gains - -- and restores parity with eager (the streaming LEFT path interleaves - -- unmatched rows differently). Streaming kept only for an unbounded probe. | isBounded leftPlan = do leftDf <- execute leftPlan rightDf <- execute rightPlan @@ -317,13 +265,11 @@ buildStream (PhysicalHashJoin jt leftKey rightKey leftPlan rightPlan) = Join.INNER -> streamingHashJoin assembleInnerBatch Join.LEFT -> streamingHashJoin assembleLeftBatch _ -> do - -- Blocking fallback for RIGHT / FULL_OUTER leftDf <- execute leftPlan rightDf <- execute rightPlan materialized (performJoin jt leftKey rightKey leftDf rightDf) where streamingHashJoin assembleFn = do - -- Materialise build (right) side once and build the compact index. rightDf <- execute rightPlan let rightDf' = if leftKey == rightKey @@ -333,7 +279,6 @@ buildStream (PhysicalHashJoin jt leftKey rightKey leftPlan rightPlan) = csSet = S.fromList [joinKey] rightHashes = Join.buildHashColumn [joinKey] rightDf' ci = Join.buildCompactIndex rightHashes - -- Stream probe (left) side batch by batch. leftStream <- buildStream leftPlan return . Stream $ do mBatch <- pullBatch leftStream @@ -346,7 +291,6 @@ buildStream (PhysicalHashJoin jt leftKey rightKey leftPlan rightPlan) = assembleLeftBatch csSet probeBatch rightDf' probeIxs buildIxs = let batchN = Core.nRows probeBatch - -- Mark which probe rows were matched (may have duplicates — that's fine). matched = VU.accumulate (\_ b -> b) @@ -358,8 +302,6 @@ buildStream (PhysicalHashJoin jt leftKey rightKey leftPlan rightPlan) = in Join.assembleLeft csSet probeBatch rightDf' allProbeIxs allBuildIxs assembleInnerBatch = Join.assembleInner - --- SortMergeJoin (blocking on both sides) ------------------------------------- buildStream (PhysicalSortMergeJoin jt leftKey rightKey leftPlan rightPlan) = do leftDf <- execute leftPlan rightDf <- execute rightPlan @@ -417,25 +359,21 @@ isStreamableAgg :: E.UExpr -> Bool isStreamableAgg (E.UExpr (E.Agg (E.CollectAgg _ _) _)) = False isStreamableAgg (E.UExpr (E.Agg (E.FoldAgg _ Nothing (_ :: a -> b -> a)) _)) = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> True -- self-merging: min, max, sum + Just Refl -> True Nothing -> False isStreamableAgg (E.UExpr (E.Agg (E.FoldAgg _ (Just _) (_ :: a -> b -> a)) _)) = case testEquality (typeRep @a) (typeRep @Int) of - Just Refl -> True -- seeded Int fold (old-style count): merge by sum + Just Refl -> True Nothing -> case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> True -- seeded self-merging + Just Refl -> True Nothing -> False isStreamableAgg (E.UExpr (E.Agg (E.MergeAgg{}) _)) = True isStreamableAgg _ = False -{- | Build the partial, merge, and finalizer plan for a list of streamable -aggregate expressions. - -* @partialAggs@ — applied per batch, producing one row per group -* @mergeAggs@ — applied when combining two partial-result DataFrames -* @finalizer@ — post-process after all batches (needed for 'MergeAgg' - where the accumulator type differs from the output type) +{- | Build the (partial, merge, finalizer) plan for a list of streamable +aggregates: @partialAggs@ run per batch, @mergeAggs@ combine two partial +results, and @finalizer@ post-processes (for 'MergeAgg' acc≠output types). -} buildAggPlan :: [(T.Text, E.UExpr)] -> @@ -451,7 +389,6 @@ buildAggPlan aggs = foldl combine ([], [], id) (map processAgg aggs) (T.Text, E.UExpr) -> ([(T.Text, E.UExpr)], [(T.Text, E.UExpr)], D.DataFrame -> D.DataFrame) processAgg (name, ue) = case ue of - -- Seedless FoldAgg: min, max, sum (self-merging when a = b) E.UExpr (E.Agg (E.FoldAgg n Nothing (f :: a -> b -> a)) (_ :: E.Expr b)) -> case testEquality (typeRep @a) (typeRep @b) of Just Refl -> @@ -460,7 +397,6 @@ buildAggPlan aggs = foldl combine ([], [], id) (map processAgg aggs) , id ) Nothing -> - -- a /= b but a = Int: merge by sum (backward compat) case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> ( [(name, ue)] @@ -474,7 +410,6 @@ buildAggPlan aggs = foldl combine ([], [], id) (map processAgg aggs) , id ) Nothing -> ([(name, ue)], [(name, ue)], id) - -- Seeded FoldAgg: old-style count (a = Int) E.UExpr (E.Agg (E.FoldAgg n (Just _) (f :: a -> b -> a)) (_ :: E.Expr b)) -> case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> @@ -496,10 +431,6 @@ buildAggPlan aggs = foldl combine ([], [], id) (map processAgg aggs) , id ) Nothing -> ([(name, ue)], [(name, ue)], id) - -- MergeAgg: count, mean, etc. - -- Partial step: accumulate into acc type (using id as finalizer). - -- Merge step: apply merge function to two acc-typed partial results. - -- Finalizer: apply fin to convert acc column to output type. E.UExpr ( E.Agg ( E.MergeAgg @@ -571,31 +502,20 @@ executeParquetScan path cfg = do -- CSV scan implementation -- --------------------------------------------------------------------------- -{- | CSV scan, SIMD-parallel. - -The file is read once into memory, split at newline boundaries into N -ByteString slices (N = RTS capabilities), and each slice is parsed in -parallel with the SIMD reader from "DataFrame.IO.CSV.Fast" via the -in-memory entry point — no temp-file roundtrip. The resulting per-chunk -DataFrames are sliced into batches and a dedicated thread feeds them -into a bounded queue. Pushdown predicates are applied per batch by the -consumer. +{- | SIMD-parallel CSV scan: the file is split at newline boundaries into one +slice per capability, parsed concurrently, sliced into batches, and fed through +a bounded queue. Pushdown predicates are applied per batch by the consumer. -} executeCsvScan :: FilePath -> Char -> CsvReader -> ScanConfig -> IO Stream executeCsvScan path _sep reader cfg = do nCaps <- getNumCapabilities chunkPaths <- splitCsvAtNewlines (max 1 nCaps) path - -- Each chunk parses in parallel via the reader carried on the - -- 'CsvSource' plan node. Parsing and queue-feeding stay disjoint to - -- avoid 14 producers all hammering a shared TBQueue (STM contention - -- dominates throughput). let schema = scanSchema cfg batchSz = scanBatchSize cfg chunkDfs <- mapConcurrently (reader schema) chunkPaths mapM_ removeFile chunkPaths - -- Bounded queue with a single writer, N concurrent readers. queue <- newTBQueueIO (fromIntegral (max 4 (2 * nCaps))) _ <- forkIO $ do forM_ chunkDfs $ \df -> @@ -606,7 +526,6 @@ executeCsvScan path _sep reader cfg = do ( do mb <- atomically (readTBQueue queue) case mb of - -- Re-insert the sentinel so repeated pulls after EOF stay Nothing. Nothing -> atomically (writeTBQueue queue Nothing) >> return Nothing Just df -> let df' = case scanPushdownPredicate cfg of @@ -663,12 +582,9 @@ sliceIntoBatches n df = starts = [0, n .. total - 1] in [Sub.range (s, min (s + n) total) df | s <- starts] -{- | Split a CSV file at newline boundaries into @n@ temp files, each -carrying the original header followed by an aligned-at-newlines slice -of the body. Returns the temp file paths; the caller is responsible -for removing them after use. The path-based 'fastReadCsvWithSchema' -mmap's each file, so we get OS-paged reads instead of a single -monolithic 'BS.readFile' of the whole input. +{- | Split a CSV file at newline boundaries into @n@ temp files, each with the +original header plus a body slice. Returns the paths (caller removes them); the +per-file reader mmap's each, giving OS-paged reads instead of one monolithic read. -} splitCsvAtNewlines :: Int -> FilePath -> IO [FilePath] splitCsvAtNewlines n path = do @@ -700,16 +616,9 @@ splitCsvAtNewlines n path = do -- Join helper -- --------------------------------------------------------------------------- -{- | Route join to the existing Operations.Join implementation. -When the left and right key names differ, rename the right key before joining. - -'Join.join' retains its first 'DataFrame' argument and makes the second one -optional, so the lazy left sub-query ('leftDf') must be passed first for LEFT -and RIGHT joins to retain the side the caller means. INNER and FULL_OUTER are -symmetric in which rows survive, and 'Operations.Join' orders their output -columns with the renamed right frame first, so they keep the @rightDf leftDf@ -order. (Passing @rightDf@ first for LEFT/RIGHT was the source of a silent -left/right inversion: a left join dropped the left side's unmatched rows.) +{- | Route a join to 'Operations.Join', renaming the right key when the names +differ. 'Join.join' keeps its first argument, so LEFT/RIGHT must pass 'leftDf' +first to retain the intended side; symmetric INNER/FULL_OUTER pass @rightDf@ first. -} performJoin :: Join.JoinType -> T.Text -> T.Text -> D.DataFrame -> D.DataFrame -> D.DataFrame @@ -727,15 +636,9 @@ performJoin jt leftKey rightKey leftDf rightDf = -- Sort order conversion -- --------------------------------------------------------------------------- -{- | Convert a plan-level @(column, direction)@ into a Permutation 'SortOrder'. - -The lazy plan only carries the column name, but 'Perm.sortBy' recovers the -'Ord' dictionary from the type parameter of @E.Col \@a@ (it returns @EQ@ for -every row when that type does not match the column's stored element type). -We therefore read the materialised column's element type and emit @E.Col@ at -exactly that type so the comparator dispatches correctly. Unknown / non-'Ord' -element types fall back to comparing as 'T.Text' (a no-op, matching the prior -behaviour) rather than failing. +{- | Convert a plan-level @(column, direction)@ into a Permutation 'SortOrder', +emitting @E.Col@ at the materialised column's element type so 'Perm.sortBy' +dispatches its comparator correctly; unknown/non-'Ord' types fall back to 'T.Text'. -} toPermSortOrder :: D.DataFrame -> (T.Text, SortOrder) -> Perm.SortOrder toPermSortOrder df (col, dir) = @@ -748,8 +651,6 @@ toPermSortOrder df (col, dir) = Ascending -> Perm.Asc (E.Col @a col) Descending -> Perm.Desc (E.Col @a col) - -- Match the column's element type against the orderable types the - -- comparator can dispatch on, emitting E.Col at the matching type. dispatch :: C.Column -> Perm.SortOrder dispatch column = case column of C.PackedText{} -> mk @T.Text @@ -774,8 +675,6 @@ toPermSortOrder df (col, dir) = tryT @b @Char $ tryT @b @T.Text (mk @T.Text) - -- If @b@ equals the candidate orderable type @a@, build the SortOrder at - -- @a@; otherwise defer to the fallback. tryT :: forall b a. (Typeable b, C.Columnable a, Ord a) => diff --git a/dataframe-lazy/src/DataFrame/Lazy/Internal/LogicalPlan.hs b/dataframe-lazy/src/DataFrame/Lazy/Internal/LogicalPlan.hs index f21c0eb3..d92f26d7 100644 --- a/dataframe-lazy/src/DataFrame/Lazy/Internal/LogicalPlan.hs +++ b/dataframe-lazy/src/DataFrame/Lazy/Internal/LogicalPlan.hs @@ -6,8 +6,8 @@ import qualified Data.Text as T import DataFrame.IO.CSV (CsvReader) import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.Expression as E -import DataFrame.Internal.Schema (Schema) import DataFrame.Operations.Join (JoinType) +import DataFrame.Schema (Schema) -- | Data source for a scan node. data DataSource diff --git a/dataframe-lazy/src/DataFrame/Lazy/Internal/Optimizer.hs b/dataframe-lazy/src/DataFrame/Lazy/Internal/Optimizer.hs index 34e3ef49..def747ed 100644 --- a/dataframe-lazy/src/DataFrame/Lazy/Internal/Optimizer.hs +++ b/dataframe-lazy/src/DataFrame/Lazy/Internal/Optimizer.hs @@ -7,18 +7,13 @@ import qualified Data.Map as M import qualified Data.Set as S import qualified Data.Text as T import qualified DataFrame.Internal.Expression as E -import DataFrame.Internal.Schema (Schema (..), elements) import DataFrame.Lazy.Internal.LogicalPlan import DataFrame.Lazy.Internal.PhysicalPlan +import DataFrame.Schema (Schema (..), elements) -{- | Optimise a logical plan and lower it to a physical plan. - -Rules applied bottom-up (in order): - 1. Filter fusion — merge consecutive Filter nodes into a conjunction - 2. Predicate pushdown — move Filter past Derive/Project toward Scan - 3. Dead column elim — drop Derive nodes whose output is never referenced - -After rule application @toPhysical@ selects concrete operators. +{- | Optimise a logical plan and lower it to a physical plan: fuse filters, push +predicates toward the scan, drop dead derived columns, then let @toPhysical@ +select concrete operators. -} optimize :: Int -> LogicalPlan -> PhysicalPlan optimize batchSz = @@ -63,11 +58,9 @@ andExpr = -- Rule 2: Predicate pushdown -- --------------------------------------------------------------------------- -{- | Push Filter nodes as close to the Scan as possible. - -* Past a @Derive@ when the predicate doesn't reference the derived column. -* Past a @Project@ when all predicate columns are in the projected set. -* Into @ScanConfig.scanPushdownPredicate@ when the child is a @Scan@. +{- | Push Filter nodes as close to the Scan as possible: past a @Derive@ it +doesn't reference, past a @Project@ that keeps all its columns, and into the +@Scan@'s pushdown predicate. -} pushPredicates :: LogicalPlan -> LogicalPlan pushPredicates (Filter p (Derive name expr child)) @@ -95,10 +88,9 @@ pushPredicates leaf = leaf -- Rule 3: Dead column elimination -- --------------------------------------------------------------------------- -{- | Collect every column name that is explicitly referenced somewhere in the -plan (in filter predicates, sort keys, aggregate keys, projection lists, -join keys, and derived expressions). Returns Nothing when "all columns -are needed" (i.e. no Project restricts the output). +{- | Collect every column name referenced anywhere in the plan (filters, sorts, +aggregate/join keys, projections, derived expressions). 'Nothing' means all +columns are needed (no Project restricts the output). -} referencedCols :: LogicalPlan -> Maybe (S.Set T.Text) referencedCols (Scan _ schema) = Just (S.fromList (M.keys (elements schema))) @@ -165,13 +157,11 @@ eliminateDeadColumns plan = go (referencedCols plan) plan -- Logical → Physical lowering -- --------------------------------------------------------------------------- -{- | Lower the (already-optimised) logical plan to a physical plan. - -Join strategy: always HashJoin (the executor can fall back to SortMerge -at runtime once statistics are available). +{- | Lower the (already-optimised) logical plan to a physical plan. Joins always +lower to HashJoin; the executor may fall back to SortMerge at runtime. -} toPhysical :: Int -> LogicalPlan -> PhysicalPlan --- Special case: Filter directly on a Scan → push into ScanConfig. +-- A Filter directly on a Scan is folded into the scan's pushdown predicate. toPhysical batchSz (Filter p (Scan (CsvSource path sep reader) schema)) = PhysicalScan (CsvSource path sep reader) diff --git a/dataframe-lazy/src/DataFrame/Lazy/Internal/PhysicalPlan.hs b/dataframe-lazy/src/DataFrame/Lazy/Internal/PhysicalPlan.hs index e31397cd..10c81714 100644 --- a/dataframe-lazy/src/DataFrame/Lazy/Internal/PhysicalPlan.hs +++ b/dataframe-lazy/src/DataFrame/Lazy/Internal/PhysicalPlan.hs @@ -3,9 +3,9 @@ module DataFrame.Lazy.Internal.PhysicalPlan where import qualified Data.Text as T import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.Expression as E -import DataFrame.Internal.Schema (Schema) import DataFrame.Lazy.Internal.LogicalPlan (DataSource, SortOrder) import DataFrame.Operations.Join (JoinType) +import DataFrame.Schema (Schema) -- | Scan-level configuration: batch size, separator, optional pushdowns. data ScanConfig = ScanConfig diff --git a/dataframe-lazy/src/DataFrame/Typed/Lazy.hs b/dataframe-lazy/src/DataFrame/Typed/Lazy.hs index 9f729a24..f1451636 100644 --- a/dataframe-lazy/src/DataFrame/Typed/Lazy.hs +++ b/dataframe-lazy/src/DataFrame/Typed/Lazy.hs @@ -12,12 +12,9 @@ Copyright : (c) 2024 - 2026 Michael Chavinda License : MIT Stability : experimental -Type-safe lazy query pipelines. - -This module combines the compile-time schema tracking of 'TypedDataFrame' -with the deferred execution of 'LazyDataFrame'. Queries are built as a -logical plan tree with phantom-typed schema tracking; execution is deferred -until 'run' is called. +Type-safe lazy query pipelines: compile-time schema tracking ('TypedDataFrame') +with the deferred execution of 'LazyDataFrame'. Queries build a phantom-typed +logical plan; execution is deferred until 'run'. @ {\-\# LANGUAGE DataKinds, TypeApplications, TypeOperators \#-\} @@ -83,11 +80,11 @@ import Prelude hiding (filter, take) import qualified DataFrame.Internal.Column as C import qualified DataFrame.Internal.Expression as E -import DataFrame.Internal.Schema (Schema) import DataFrame.Lazy.Internal.DataFrame (LazyDataFrame) import qualified DataFrame.Lazy.Internal.DataFrame as L import DataFrame.Lazy.Internal.LogicalPlan (SortOrder (..)) import DataFrame.Operations.Join (JoinType (..)) +import DataFrame.Schema (Schema) import DataFrame.Typed.Expr import DataFrame.Typed.Freeze (unsafeFreeze) import DataFrame.Typed.Schema diff --git a/dataframe-learn/README.md b/dataframe-learn/README.md index 8e366ed9..a3d0f688 100644 --- a/dataframe-learn/README.md +++ b/dataframe-learn/README.md @@ -1,4 +1,4 @@ - +