DataHaskell · mchav · Jul 12, 2026 · Jul 11, 2026 · Jul 11, 2026 · Jul 12, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Revision history for dataframe
 
+## 3.0.0.0
+
+A breaking release that removes internal functions from the public API.
+
+### Highlights
+* Internalish modules now have public APIs/contracts. `DataFrame.Core`, `DataFrame.Schema`, and `DataFrame.Learn`.
+* `import DataFrame` now surfaces `Columnable`, `Columnable'`, `eSize`, `NamedExpr`,
+  `SchemaType(..)`, and the `Semigroup`/`Monoid DataFrame` instances directly.
+
+### Breaking changes
+* Internal modules are no longer on the public surface.
+* Subpackage major bumps.
+
 ## 2.3.0.0
 
 ### Breaking changes

diff --git a/app/LazyBenchmark.hs b/app/LazyBenchmark.hs
@@ -2,21 +2,14 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE TypeApplications #-}
 
-{- | End-to-end smoke test and benchmark for the Lazy streaming API.
+{- | End-to-end smoke test and benchmark for the Lazy streaming API:
+     generate a CSV then run five Lazy queries over it.
 
      Usage:
-       cabal run lazy-bench [-- [OPTIONS]]
-
-     Options:
-       --rows N       Number of rows to generate (default: 1_000_000_000)
-       --file PATH    Output CSV path          (default: /tmp/lazy_1b.csv)
-       --skip-gen     Skip generation if the file already exists
-
-     The executable generates a CSV file (streaming, constant memory) then
-     runs five Lazy queries over it, printing timing and result summaries.
-
-     For heap/GC stats run with:
-       cabal run lazy-bench -- +RTS -s -RTS
+       cabal run lazy-bench [-- [OPTIONS]]        (+RTS -s -RTS for heap stats)
+       --rows N     rows to generate (default 1_000_000_000)
+       --file PATH  output CSV path (default /tmp/lazy_1b.csv)
+       --skip-gen   reuse the file if it already exists
 -}
 module Main where
 
@@ -26,9 +19,9 @@ import qualified Data.Map as M
 import qualified Data.Text as T
 import Data.Time (UTCTime, diffUTCTime, getCurrentTime)
 import qualified DataFrame as D
-import DataFrame.Internal.Schema (Schema (..), schemaType)
 import qualified DataFrame.Lazy as L
 import DataFrame.Operators
+import DataFrame.Schema (Schema (..), schemaType)
 import System.Directory (doesFileExist, getFileSize)
 import System.Environment (getArgs)
 import System.Exit (exitFailure)
@@ -185,9 +178,6 @@ main = do
         n = optRows opts
         pathT = T.pack path
 
-    -- -----------------------------------------------------------------------
-    -- Phase 1: Generate
-    -- -----------------------------------------------------------------------
     putStrLn "=== Lazy API 1B-row benchmark ==="
     putStrLn $ "    rows   : " ++ commas n
     putStrLn $ "    file   : " ++ path
@@ -205,12 +195,8 @@ main = do
             sz <- getFileSize path
             putStrLn $ "  file size: " ++ showBytes sz
 
-    -- -----------------------------------------------------------------------
-    -- Phase 2: Lazy queries
-    -- -----------------------------------------------------------------------
     putStrLn "\nPhase 2: Lazy queries"
 
-    -- Schema for the generated CSV: id (Int), x (Double), y (Double), category (Text)
     let schema =
             Schema $
                 M.fromList
@@ -220,25 +206,17 @@ main = do
                     , ("category", schemaType @T.Text)
                     ]
 
-    -- Q1: Preview — limit 20, no filter.
-    -- Demonstrates that the executor reads only the first batch.
     runQuery "Q1 — preview first 20 rows (no filter)" $
         L.runDataFrame $
             L.take 20 $
                 L.scanCsv schema pathT
 
-    -- Q2: Filter + limit.
-    -- x > 0.999 ≈ 0.1% of rows. With a 512K-row batch the executor finds
-    -- ~512 matches in the first batch and stops — reads only one batch.
     runQuery "Q2 — filter (x > 0.999), limit 20" $
         L.runDataFrame $
             L.take 20 $
                 L.filter (col @Double "x" .> lit (0.999 :: Double)) $
                     L.scanCsv schema pathT
 
-    -- Q3: Filter + derive + select + limit.
-    -- Shows projection pushdown: only id/x/y/category are read, z is derived.
-    -- Predicate pushdown moves the filter into the scan batch loop.
     runQuery "Q3 — filter (x > 0.999), derive z = x*y, select [id,z], limit 20" $
         L.runDataFrame $
             L.take 20 $
@@ -247,21 +225,13 @@ main = do
                         L.filter (col @Double "x" .> lit (0.999 :: Double)) $
                             L.scanCsv schema pathT
 
-    -- Q4: Filter fusion demo.
-    -- Two consecutive filters are fused into one AND predicate by the optimizer.
-    -- Result: rows where x > 0.5 AND y > 0.5 (≈ 25% of total).
-    -- We limit to keep result size manageable.
     runQuery "Q4 — filter fusion: (x > 0.5) . (y > 0.5), limit 20" $
         L.runDataFrame $
             L.take 20 $
                 L.filter (col @Double "y" .> lit (0.5 :: Double)) $
                     L.filter (col @Double "x" .> lit (0.5 :: Double)) $
                         L.scanCsv schema pathT
 
-    -- Q5: Full scan, heavy filter, count results.
-    -- x > 0.999 across the whole file ≈ 0.1% × N rows.
-    -- For 1B rows that is ~1M results — materialised into one DataFrame.
-    -- This query exercises streaming across all batches.
     runQuery
         ( "Q5 — full scan, filter (x > 0.999), count (~"
             ++ approx (n `div` 1000)

diff --git a/app/Synthesis.hs b/app/Synthesis.hs
@@ -47,8 +47,8 @@ main = do
             D.randomSplit (mkStdGen 4232) 0.7 (DT.thaw (clean train))
         testDf = DT.thaw (clean test)
 
-        model =
-            fitDecisionTree
+        fitted =
+            fit
                 ( defaultTreeConfig
                     { maxTreeDepth = 5
                     , minSamplesSplit = 5
@@ -68,8 +68,9 @@ main = do
                 )
                 (F.fromMaybe 0 (F.col @(Maybe Int) "Survived"))
                 (trainDf |> D.exclude ["PassengerId"])
+        model = predict fitted
 
-    print model
+    print fitted
 
     putStrLn "Training accuracy: "
     print $ computeAccuracy (trainDf |> D.derive (F.name prediction) model)

diff --git a/cabal.project.bare b/cabal.project.bare
@@ -20,3 +20,4 @@ packages:
     dataframe-viz
     dataframe-learn
     dataframe-lazy
+    dataframe-expr-serializer
diff --git a/dataframe-arrow/dataframe-arrow.cabal b/dataframe-arrow/dataframe-arrow.cabal
@@ -56,10 +56,10 @@ foreign-library dataframe-arrow
     other-modules:    DataFrame.FFI
     build-depends:
         base                  >= 4   && < 5,
-        dataframe-core        ^>= 1.1,
-        dataframe-learn       ^>= 1.1,
-        dataframe:arrow-bridge >= 1   && < 3,
-        dataframe-fastcsv     ^>= 1.1.1,
+        dataframe-core:internal ^>= 2.0,
+        dataframe-learn       ^>= 2.0,
+        dataframe:arrow-bridge >= 1   && < 4,
+        dataframe-fastcsv     ^>= 1.2,
         text                  >= 2.1 && < 3,
         aeson                 >= 0.11 && < 3,
         bytestring            >= 0.11 && < 0.13,

diff --git a/dataframe-arrow/ffi-export/DataFrame/FFI.hs b/dataframe-arrow/ffi-export/DataFrame/FFI.hs
@@ -36,7 +36,8 @@ import DataFrame.DecisionTree (
     defaultColumnOrdering,
     defaultSynthConfig,
     defaultTreeConfig,
-    fitDecisionTree,
+    fit,
+    predict,
  )
 import DataFrame.IO.Arrow (dataframeToArrow)
 import DataFrame.IO.CSV.Fast (fastReadCsvWithSchema)
@@ -49,14 +50,9 @@ import DataFrame.Internal.Expression (Expr (Col))
 foreign export ccall "dfExecutePlan"
     dfExecutePlan :: CString -> Ptr Word64 -> Ptr Word64 -> IO CInt
 
-{- | Execute a JSON-encoded query plan and return Arrow C Data Interface
-  pointers to Python via output parameters.
-
-  @planCS@    – JSON plan string (see DataFrame.IR for schema)
-  @schemaOut@ – receives the ArrowSchema* address as a Word64
-  @arrayOut@  – receives the ArrowArray*  address as a Word64
-
-  Returns 0 on success, -1 on error.
+{- | Execute a JSON-encoded query plan, returning Arrow C Data Interface
+pointers to Python via the @schemaOut@/@arrayOut@ output parameters (each an
+address as a 'Word64'). Returns 0 on success, -1 on error.
 -}
 dfExecutePlan :: CString -> Ptr Word64 -> Ptr Word64 -> IO CInt
 dfExecutePlan planCS schemaOut arrayOut = do
@@ -88,22 +84,9 @@ foreign export ccall "dfFitDecisionTree"
 foreign export ccall "dfFreeModel"
     dfFreeModel :: Ptr CChar -> IO ()
 
-{- | Fit a TAO decision tree on the materialized result of @plan_json@, predicting
-  @target_col@ from the remaining columns.
-
-  The caller owns the returned buffer and must release it with 'dfFreeModel'.
-
-  @target_type@ may be the literal string @"auto"@ to infer the target's type
-  from the materialized DataFrame's schema, or one of the wire-format type
-  tags (@"int"@, @"double"@, @"text"@, @"bool"@, …).
-
-  @config_json@ is a JSON object with the serializable TreeConfig fields:
-  @max_depth@, @min_samples_split@, @min_leaf_size@, @percentiles@,
-  @expression_pairs@, @tao_iterations@, @tao_convergence_tol@,
-  @max_expr_depth@, @bool_expansion@, @complexity_penalty@,
-  @enable_string_ops@, @enable_cross_cols@, @enable_arith_ops@.
-
-  Returns 0 on success, -1 on error.
+{- | Fit a TAO decision tree on a plan's materialized result, predicting the
+target column from the rest (@target_type@: @"auto"@ or a wire type tag;
+@config_json@: 'TreeConfig' fields). Caller frees via 'dfFreeModel'; 0 ok, -1 err.
 -}
 dfFitDecisionTree ::
     CString ->
@@ -255,31 +238,31 @@ inferTargetType target df = dispatchType (columnTypeRep (unsafeGetColumn target
 fitTreeWithType ::
     T.Text -> TreeConfig -> T.Text -> DataFrame -> IO BS.ByteString
 fitTreeWithType ttag cfg target df = case ttag of
-    "int" -> fit @Int
-    "int8" -> fit @Int8
-    "int16" -> fit @Int16
-    "int32" -> fit @Int32
-    "int64" -> fit @Int64
-    "word" -> fit @Word
-    "word8" -> fit @Word8
-    "word16" -> fit @Word16
-    "word32" -> fit @Word32
-    "word64" -> fit @Word64
-    "integer" -> fit @Integer
-    "double" -> fit @Double
-    "float" -> fit @Float
-    "bool" -> fit @Bool
-    "char" -> fit @Char
-    "text" -> fit @T.Text
-    "string" -> fit @String
+    "int" -> fitEncode @Int
+    "int8" -> fitEncode @Int8
+    "int16" -> fitEncode @Int16
+    "int32" -> fitEncode @Int32
+    "int64" -> fitEncode @Int64
+    "word" -> fitEncode @Word
+    "word8" -> fitEncode @Word8
+    "word16" -> fitEncode @Word16
+    "word32" -> fitEncode @Word32
+    "word64" -> fitEncode @Word64
+    "integer" -> fitEncode @Integer
+    "double" -> fitEncode @Double
+    "float" -> fitEncode @Float
+    "bool" -> fitEncode @Bool
+    "char" -> fitEncode @Char
+    "text" -> fitEncode @T.Text
+    "string" -> fitEncode @String
     other ->
         ioError . userError $
             "DataFrame.FFI.fitTreeWithType: unsupported target type tag: "
                 ++ T.unpack other
   where
-    fit :: forall a. (Columnable a, Ord a) => IO BS.ByteString
-    fit = do
-        let expr = fitDecisionTree @a cfg (Col @a target) df
+    fitEncode :: forall a. (Columnable a, Ord a) => IO BS.ByteString
+    fitEncode = do
+        let expr = predict (fit cfg (Col @a target) df)
         case encodeExprToBytes expr of
             Right bs -> return bs
             Left err -> ioError (userError $ "encodeExprToBytes: " ++ err)
diff --git a/dataframe-core/dataframe-core.cabal b/dataframe-core/dataframe-core.cabal
@@ -1,6 +1,6 @@
-cabal-version:      2.4
+cabal-version:      3.4
 name:               dataframe-core
-version:            1.1.1.0
+version:            2.0.0.0
 synopsis:           Core data structures for the dataframe library.
 description:
     Minimal interchange-format types for the @dataframe@ ecosystem:
@@ -28,8 +28,9 @@ common warnings
         -Wunused-local-binds
         -Wunused-packages
 
-library
+library internal
     import:             warnings
+    visibility:         public
     exposed-modules:
                         DataFrame.Errors
                         DataFrame.Operators
@@ -61,17 +62,30 @@ library
                         DataFrame.Internal.Simplify
                         DataFrame.Internal.Types
                         DataFrame.Internal.Utf8
+    build-depends:      base >= 4 && < 5,
+                        containers >= 0.6.7 && < 0.9,
+                        primitive >= 0.7 && < 0.10,
+                        random >= 1 && < 2,
+                        text >= 2.1 && < 3,
+                        vector ^>= 0.13
+    hs-source-dirs:     src-internal
+    default-language:   Haskell2010
+
+library
+    import:             warnings
+    exposed-modules:
+                        DataFrame.Core
                         DataFrame.Typed.Freeze
                         DataFrame.Typed.Generic
                         DataFrame.Typed.Record
                         DataFrame.Typed.Schema
                         DataFrame.Typed.Types
                         DataFrame.Typed.Util
+    reexported-modules: DataFrame.Errors,
+                        DataFrame.Operators
     build-depends:      base >= 4 && < 5,
-                        containers >= 0.6.7 && < 0.9,
-                        primitive >= 0.7 && < 0.10,
-                        random >= 1 && < 2,
                         text >= 2.1 && < 3,
-                        vector ^>= 0.13
+                        vector ^>= 0.13,
+                        dataframe-core:internal
     hs-source-dirs:     src
     default-language:   Haskell2010
diff --git a/...src/DataFrame/Display/Terminal/Colours.hs → ...nal/DataFrame/Display/Terminal/Colours.hs b/...src/DataFrame/Display/Terminal/Colours.hs → ...nal/DataFrame/Display/Terminal/Colours.hs
diff --git a/...DataFrame/Display/Terminal/PrettyPrint.hs → ...DataFrame/Display/Terminal/PrettyPrint.hs b/...DataFrame/Display/Terminal/PrettyPrint.hs → ...DataFrame/Display/Terminal/PrettyPrint.hs
@@ -14,19 +14,18 @@ data RenderFormat = Plain | Markdown
 
 -- Utility functions to show a DataFrame as a Markdown-ish table.
 
--- Adapted from: https://stackoverflow.com/questions/5929377/format-list-output-in-haskell
--- a type for fill functions
+-- Fill functions, adapted from:
+-- https://stackoverflow.com/questions/5929377/format-list-output-in-haskell
 type Filler = Int -> T.Text -> T.Text
 
--- a type for describing table columns
+-- A description of one table column.
 data ColDesc t = ColDesc
     { colTitleFill :: Filler
     , colTitle :: T.Text
     , colValueFill :: Filler
     }
 
--- functions that fill a string (s) to a given width (n) by adding pad
--- character (c) to align left, right, or center
+-- Fill text to a given width with a pad character, aligning left, right, or center.
 fillLeft :: Char -> Int -> T.Text -> T.Text
 fillLeft c n s = s <> T.replicate (n - T.length s) (T.singleton c)
 
@@ -41,7 +40,7 @@ fillCenter c n s =
     l = x `div` 2
     r = x - l
 
--- functions that fill with spaces
+-- Fill with spaces.
 left :: Int -> T.Text -> T.Text
 left = fillLeft ' '
 
@@ -63,9 +62,6 @@ showTable ::
     T.Text
 showTable fmt header types columns =
     let isMarkdown = fmt == Markdown
-        -- In a GitHub pipe table a literal '|' ends the cell and a newline ends
-        -- the row, so a value like the operator name <|> would split the row into
-        -- the wrong columns. Escape both for the Markdown format only.
         esc = if isMarkdown then escapeMarkdownCell else id
         hdr = map esc header
         tys = map esc types

diff --git a/dataframe-core/src/DataFrame/Errors.hs → ...ame-core/src-internal/DataFrame/Errors.hs b/dataframe-core/src/DataFrame/Errors.hs → ...ame-core/src-internal/DataFrame/Errors.hs