diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8eedc3a4504e8..6fc96d4231c83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -106,7 +106,6 @@ /src/repr/src/row @MaterializeInc/persist /src/repr-test-util @MaterializeInc/cluster /src/rocksdb @MaterializeInc/cluster -/src/s3-datagen @MaterializeInc/cluster /src/secrets @MaterializeInc/cloud /src/segment /src/service @MaterializeInc/cluster diff --git a/Cargo.lock b/Cargo.lock index 8a488ba4b23ff..b3d0c2976e19f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1869,15 +1869,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" -[[package]] -name = "bytefmt" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "590b1af059a21c47d4da7cd11f05e08b1992b58b5b4acf2a5e10d7e53aed3d74" -dependencies = [ - "regex", -] - [[package]] name = "bytemuck" version = "1.25.2" @@ -6968,23 +6959,13 @@ dependencies = [ name = "mz-durable-cache" version = "0.0.0" dependencies = [ - "async-trait", - "bytes", "differential-dataflow", - "futures", - "itertools 0.14.0", - "mz-dyncfg", "mz-ore", "mz-persist-client", "mz-persist-types", - "mz-timely-util", - "prometheus", - "prost", - "serde", "timely", "tokio", "tracing", - "uuid", ] [[package]] @@ -8309,24 +8290,6 @@ dependencies = [ "timely", ] -[[package]] -name = "mz-s3-datagen" -version = "0.0.0" -dependencies = [ - "anyhow", - "aws-config", - "aws-sdk-s3", - "bytefmt", - "clap", - "futures", - "indicatif", - "mz-aws-util", - "mz-ore", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "mz-secrets" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index e72662ef8b42b..bb081dea6479c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,6 @@ members = [ "src/kafka-util", "src/license-keys", "src/materialized", - "src/materialized", "src/metabase", "src/metrics", "src/metrics-catalog", @@ -93,7 +92,6 @@ members = [ "src/rocksdb", "src/rocksdb-types", "src/row-spine", - "src/s3-datagen", "src/secrets", "src/segment", "src/server-core", @@ -220,7 +218,6 @@ default-members = [ "src/rocksdb", "src/rocksdb-types", "src/row-spine", - "src/s3-datagen", "src/secrets", "src/segment", "src/server-core", @@ -312,7 +309,6 @@ base64 = "0.22.1" bincode = "1.3.3" bitflags = "1.3.2" buildid = "1.0.4" -bytefmt = "0.1.7" bytemuck = { version = "1.23.1", features = ["extern_crate_alloc", "latest_stable_rust"] } byteorder = "1.5" bytes = "1.11.1" @@ -346,7 +342,6 @@ dec = "0.4.9" derivative = "2.2.0" differential-dataflow = "0.25.0" differential-dogs3 = "0.25.0" -digest = "0.10.7" dirs = "6.0.0" duckdb = { version = "1.4.3", default-features = false, features = ["native-tls"] } dynfmt = { version = "0.1.5", features = ["curly"] } @@ -376,7 +371,6 @@ hickory-resolver = "0.26.1" hmac = "0.12.1" http = "1.4.0" http-body-util = "0.1.3" -httparse = "1.8.0" humantime = "2.3.0" hyper = { version = "1.9.0", features = ["http1", "server"] } # hyper 0.14 is used by AWS SDK. Used to override the DNS resolver used by the SDK, @@ -513,7 +507,6 @@ ssh-key = "0.6.7" stacker = "0.1.24" static_assertions = "1.1" strsim = "0.11.1" -subtle = "2.6.1" supports-color = "3.0.2" syn = { version = "2.0.119", features = ["extra-traits", "full", "parsing", "printing"] } sysctl = "0.7.1" diff --git a/console/img/github-mark-white.svg b/console/img/github-mark-white.svg deleted file mode 100644 index bab151dddbb86..0000000000000 --- a/console/img/github-mark-white.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/console/img/github-mark.svg b/console/img/github-mark.svg deleted file mode 100644 index 0c8059f3ea646..0000000000000 --- a/console/img/github-mark.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/console/img/google-mark.svg b/console/img/google-mark.svg deleted file mode 100644 index 51237508a5af5..0000000000000 --- a/console/img/google-mark.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/console/public/logo.png b/console/public/logo.png deleted file mode 100644 index a5314daf42795..0000000000000 Binary files a/console/public/logo.png and /dev/null differ diff --git a/console/src/api/fronteggToken.ts b/console/src/api/fronteggToken.ts deleted file mode 100644 index 9f45a0fb78cc5..0000000000000 --- a/console/src/api/fronteggToken.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import * as Sentry from "@sentry/react"; - -import { ContextHolder } from "~/external-library-wrappers/frontegg"; - -/** - * Returns the current Frontegg access token. - * Usually we can get the access token from the `useAuth` hook, - * but for our middlewares that aren't in the context of - * React, we want to get it straight from the Frontegg - * Redux store which is available via `ContextHolder`. - */ -export function getAccessToken() { - // Get the access token from Frontegg's context holder - const accessToken = ContextHolder.for("default").getAccessToken(); - - // The token will most likely always exist since this function - // will be called when logged in. If not, we should alert Sentry. - if (!accessToken) { - Sentry.addBreadcrumb({ - level: "error", - category: "auth", - message: "Failed to refresh auth token", - }); - } - return accessToken; -} diff --git a/console/src/api/materialize/useCreditConsumption.ts b/console/src/api/materialize/useCreditConsumption.ts deleted file mode 100644 index 7a7d309d029f4..0000000000000 --- a/console/src/api/materialize/useCreditConsumption.ts +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { sql } from "kysely"; -import React from "react"; - -import { queryBuilder, useSqlTyped } from "~/api/materialize"; - -/** - * Gets the number of compute credits per hour that are being consumed. - * @param since filters replicas dropped before the given date, or undefined which - * returns all replicas that are still running. - */ -export function useCreditConsumption(since?: Date) { - const query = React.useMemo(() => { - let qb = queryBuilder - .selectFrom("mz_cluster_replica_history as mcrh") - .select([ - "mcrh.replica_id as replicaId", - "mcrh.size", - "mcrh.created_at as createdAt", - "mcrh.dropped_at as dropppedAt", - sql`mcrh.credits_per_hour`.as("creditsPerHour"), - ]); - if (since) { - qb = qb.where((eb) => - eb.or([ - eb("mcrh.dropped_at", "is", null), - eb("mcrh.dropped_at", ">=", since), - ]), - ); - } else { - qb = qb.where("mcrh.dropped_at", "is", null); - } - return qb.compile(); - }, [since]); - - const response = useSqlTyped(query); - let results = null; - if (response.results) { - results = response.results.reduce((acc, h) => acc + h.creditsPerHour, 0); - } - return { ...response, results }; -} diff --git a/console/src/api/materialize/useShowCreate.ts b/console/src/api/materialize/useShowCreate.ts deleted file mode 100644 index 74fc977c0f9d4..0000000000000 --- a/console/src/api/materialize/useShowCreate.ts +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { sql } from "kysely"; -import React from "react"; - -import { - buildFullyQualifiedObjectName, - queryBuilder, - SchemaObject, - useSqlTyped, -} from "~/api/materialize"; - -export type DDLNoun = "SINK" | "SOURCE"; - -/** - * Fetches the DDL statement for creating a schema object - */ -function useShowCreate(noun: DDLNoun, schemaObject?: SchemaObject) { - const query = React.useMemo(() => { - if (!schemaObject) return null; - - return sql<{ - name: string; - create_sql: string; - }>`SHOW CREATE ${sql.raw(noun)} ${buildFullyQualifiedObjectName( - schemaObject, - )}`.compile(queryBuilder); - }, [noun, schemaObject]); - - const response = useSqlTyped(query); - - let ddl: string | null = null; - if (response.results) { - ddl = response.results[0].create_sql; - } - - return { ...response, results: ddl }; -} - -export default useShowCreate; diff --git a/console/src/components/InternalOnlyNotice.tsx b/console/src/components/InternalOnlyNotice.tsx deleted file mode 100644 index 3293545457ce0..0000000000000 --- a/console/src/components/InternalOnlyNotice.tsx +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { BoxProps, Text, Tooltip, useTheme } from "@chakra-ui/react"; -import React from "react"; - -import { MaterializeTheme } from "~/theme"; - -const InternalOnlyNotice = (props: BoxProps) => { - const { colors } = useTheme(); - return ( - - - Internal Only - - - ); -}; - -export default InternalOnlyNotice; diff --git a/console/src/components/PreviewNotice.tsx b/console/src/components/PreviewNotice.tsx deleted file mode 100644 index 813eaad70cca4..0000000000000 --- a/console/src/components/PreviewNotice.tsx +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { BoxProps, Text, Tooltip, useTheme } from "@chakra-ui/react"; -import React from "react"; - -import { MaterializeTheme } from "~/theme"; - -const PreviewNotice = (props: BoxProps) => { - const { colors } = useTheme(); - return ( - - - Preview - - - ); -}; - -export default PreviewNotice; diff --git a/console/src/hooks/useCachedQueryData.ts b/console/src/hooks/useCachedQueryData.ts deleted file mode 100644 index 5f4e03320c2d3..0000000000000 --- a/console/src/hooks/useCachedQueryData.ts +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { QueryKey, useQueryClient } from "@tanstack/react-query"; -import deepEqual from "fast-deep-equal"; -import React from "react"; - -export function useCachedQueryData(queryKey: QueryKey) { - const queryClient = useQueryClient(); - const [data, setData] = React.useState(() => - queryClient.getQueryData(queryKey), - ); - - React.useEffect(() => { - const unsubscribe = queryClient.getQueryCache().subscribe((event) => { - if (deepEqual(event.query.queryKey, queryKey)) { - // setTimeout avoids the "Cannot update a component while rendering a different component" error - setTimeout(() => { - setData(event.query.state.data); - }, 0); - } - }); - - return () => { - unsubscribe(); - }; - }, [queryClient, queryKey]); - - return data; -} diff --git a/console/src/layouts/NavBar/SectionNav.tsx b/console/src/layouts/NavBar/SectionNav.tsx deleted file mode 100644 index e224399d77b59..0000000000000 --- a/console/src/layouts/NavBar/SectionNav.tsx +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { StackProps, useTheme, VStack } from "@chakra-ui/react"; -import React from "react"; - -import { MaterializeTheme } from "~/theme"; - -export const SectionNav = (props: StackProps) => { - const { colors } = useTheme(); - - return ( - - ); -}; diff --git a/console/src/platform/auth/PasswordField.tsx b/console/src/platform/auth/PasswordField.tsx deleted file mode 100644 index 81d43ec04873b..0000000000000 --- a/console/src/platform/auth/PasswordField.tsx +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - FormControl, - FormHelperText, - Input, - InputProps, - Text, - useTheme, - VStack, -} from "@chakra-ui/react"; -import React from "react"; -import { FieldError, MultipleFieldErrors } from "react-hook-form"; - -import { LabeledInput } from "~/components/formComponentsV2"; -import { MaterializeTheme } from "~/theme"; - -import { passwordRules } from "./utils"; - -export const PasswordField = (props: { - errors: FieldError | undefined; - label?: string; - inputProps: InputProps; -}) => { - return ( - - - - - - - - - ); -}; - -export const PasswordRules = (props: { - errors: MultipleFieldErrors | undefined; -}) => { - const { colors } = useTheme(); - - return ( - - - Password must: - -
    - {Object.entries(passwordRules).map(([key, message]) => ( - - {message} - - ))} -
- - ); -}; diff --git a/console/src/platform/auth/utils.ts b/console/src/platform/auth/utils.ts deleted file mode 100644 index 10f34be13bf68..0000000000000 --- a/console/src/platform/auth/utils.ts +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { ValidateResult } from "react-hook-form"; - -export function validatePassword(): Record< - string, - (value: string) => ValidateResult -> { - return { - min: (value: string) => value.length >= 10, - uppercase: (value: string) => Boolean(value.match(/[A-Z]/)), - lowercase: (value: string) => Boolean(value.match(/[a-z]/)), - number: (value: string) => Boolean(value.match(/\d/)), - special: (value: string) => Boolean(value.match(/\W/)), - repeating: (value: string) => !value.match(/(\w)\1\1/), - }; -} - -export const passwordRules: Record< - keyof ReturnType, - string -> = { - min: "be at least 10 characters", - uppercase: "contain at least one uppercase character", - lowercase: "contain at least one lowercase character", - number: "contain at least one number", - special: "contain at least one special character", - repeating: "not contain 3 or more repeating characters", -}; diff --git a/console/src/platform/environment-not-ready/EnvironmentStatusPill.tsx b/console/src/platform/environment-not-ready/EnvironmentStatusPill.tsx deleted file mode 100644 index ef638105344c5..0000000000000 --- a/console/src/platform/environment-not-ready/EnvironmentStatusPill.tsx +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { BoxProps, Flex, Text, useTheme } from "@chakra-ui/react"; -import React from "react"; - -import { Environment } from "~/store/environments"; -import { MaterializeTheme } from "~/theme"; - -import { - getBackgroundColor, - getBorderColor, - getIcon, - getText, - getTextColor, -} from "./utils"; - -export type EnvironmentStatusPillProps = BoxProps & { - environment: Environment; - regionId: string; -}; - -const EnvironmentStatusPill = ({ - environment, - regionId, - ...boxProps -}: EnvironmentStatusPillProps) => { - const { colors } = useTheme(); - - return ( - - {getIcon(environment)} - {getText(environment, regionId)} - - ); -}; - -export default EnvironmentStatusPill; diff --git a/console/src/platform/environment-overview/constants.ts b/console/src/platform/environment-overview/constants.ts deleted file mode 100644 index c3c4c0d0a7a9c..0000000000000 --- a/console/src/platform/environment-overview/constants.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -export const TIME_PERIOD_OPTIONS = { - "60": "Last hour", - "180": "Last 3 hours", - "360": "Last 6 hours", - "1440": "Last 24 hours", -}; diff --git a/console/src/platform/roles/RemoveUserMenuItem.tsx b/console/src/platform/roles/RemoveUserMenuItem.tsx deleted file mode 100644 index 6b9ece2223447..0000000000000 --- a/console/src/platform/roles/RemoveUserMenuItem.tsx +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { MenuItem, useDisclosure, useTheme } from "@chakra-ui/react"; -import React from "react"; - -import { MaterializeTheme } from "~/theme"; - -import RemoveUserModal from "./RemoveUserModal"; - -export interface RemoveUserMenuItemProps { - roleName: string; - memberName: string; - onSuccess?: () => void; -} - -const RemoveUserMenuItem = ({ - roleName, - memberName, - onSuccess, -}: RemoveUserMenuItemProps) => { - const { isOpen, onOpen, onClose } = useDisclosure(); - const { colors } = useTheme(); - - return ( - <> - { - e.stopPropagation(); - onOpen(); - }} - color={colors.accent.red} - > - Remove user - - {isOpen && ( - - )} - - ); -}; - -export default RemoveUserMenuItem; diff --git a/console/src/svg/CrosshairIcon.tsx b/console/src/svg/CrosshairIcon.tsx deleted file mode 100644 index 9109335829a59..0000000000000 --- a/console/src/svg/CrosshairIcon.tsx +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { Icon, IconProps, useTheme } from "@chakra-ui/react"; -import React from "react"; - -import { MaterializeTheme } from "~/theme"; - -const CrosshairIcon = (props: IconProps) => { - const { colors } = useTheme(); - return ( - - - - - - - - ); -}; - -export default CrosshairIcon; diff --git a/console/src/svg/DiamondErrorIcon.tsx b/console/src/svg/DiamondErrorIcon.tsx deleted file mode 100644 index 19ccc8a417a12..0000000000000 --- a/console/src/svg/DiamondErrorIcon.tsx +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import React from "react"; - -const DiamondErrorIcon = () => ( - - - - - - - - - - - - -); - -export default DiamondErrorIcon; diff --git a/console/src/svg/DoubleChevronRightIcon.tsx b/console/src/svg/DoubleChevronRightIcon.tsx deleted file mode 100644 index a5d6758cdf1d1..0000000000000 --- a/console/src/svg/DoubleChevronRightIcon.tsx +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { Icon, IconProps } from "@chakra-ui/react"; -import React from "react"; - -const DoubleChevronRightIcon = (props: IconProps) => ( - - - - -); - -export default DoubleChevronRightIcon; diff --git a/console/src/svg/PlusCircleIcon.tsx b/console/src/svg/PlusCircleIcon.tsx deleted file mode 100644 index 16f92faddfa2d..0000000000000 --- a/console/src/svg/PlusCircleIcon.tsx +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import React from "react"; - -const PlusCircleIcon = () => ( - - - - - - - - - - - - -); -export default PlusCircleIcon; diff --git a/console/src/svg/SearchIcon.tsx b/console/src/svg/SearchIcon.tsx deleted file mode 100644 index 3ffde7918c7f8..0000000000000 --- a/console/src/svg/SearchIcon.tsx +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import React from "react"; - -const SearchIcon = (props: { color: string }) => ( - - - - -); - -export default SearchIcon; diff --git a/console/src/svg/ShieldIcon.tsx b/console/src/svg/ShieldIcon.tsx deleted file mode 100644 index 88c05d30b9fb3..0000000000000 --- a/console/src/svg/ShieldIcon.tsx +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import React from "react"; - -const ShieldIcon = () => ( - - - - - -); - -export default ShieldIcon; diff --git a/console/src/theme/components/IconButton.ts b/console/src/theme/components/IconButton.ts deleted file mode 100644 index f5d532d019656..0000000000000 --- a/console/src/theme/components/IconButton.ts +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -// IconButton uses the Button component styles, don't try to override it here. -// https://github.com/chakra-ui/chakra-ui/issues/3746 diff --git a/misc/dist/deb-scripts/postinst b/misc/dist/deb-scripts/postinst deleted file mode 100755 index d88d236184ce4..0000000000000 --- a/misc/dist/deb-scripts/postinst +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -e - -# The weird username is intended to be unlikely -# to conflict with real local users. -adduser --system --group --force-badname --no-create-home --quiet _Materialize -mkdir -p /var/lib/materialize/ -chown _Materialize:_Materialize /var/lib/materialize/ - -if [ -x /bin/systemctl ]; then - systemctl daemon-reload >/dev/null 2>&1 || true -fi diff --git a/misc/dist/materialized.service b/misc/dist/materialized.service deleted file mode 100644 index 0fb6dcb9fb36a..0000000000000 --- a/misc/dist/materialized.service +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -[Unit] -Description=Materialize streaming database -After=network.target - -[Service] -Type=exec -User=_Materialize -Group=_Materialize - -ExecStart=/usr/bin/materialized --data-directory /var/lib/materialize/mzdata - -[Install] -WantedBy=multi-user.target diff --git a/misc/python/materialize/build_config.py b/misc/python/materialize/build_config.py deleted file mode 100644 index ea84b6703baa8..0000000000000 --- a/misc/python/materialize/build_config.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import os -from pathlib import Path -from textwrap import dedent -from typing import Any - -import toml - - -class LocalState: - """Local state persisted by a tool. - - Users should not expect this state to be durable, it can be blown away at - at point. - - Stored at: ~/.cache/materialize/build_state.toml - """ - - def __init__(self, path: Path): - self.path = path - if path.is_file(): - with open(path) as f: - self.data = toml.load(f) - else: - self.data = {} - - @staticmethod - def default_path() -> Path: - home = Path.home() - path = home / ".cache" / "materialize" / "build_state.toml" - return path - - @classmethod - def read(cls, namespace: str) -> Any | None: - cache = LocalState(LocalState.default_path()) - return cache.data.get(namespace, None) - - @classmethod - def write(cls, namespace: str, val: Any): - cache = LocalState(LocalState.default_path()) - cache.data[namespace] = val - - Path(os.path.dirname(cache.path)).mkdir(parents=True, exist_ok=True) - with open(cache.path, "w+") as f: - toml.dump(cache.data, f) - - -class TeleportLocalState: - def __init__(self, data: dict[str, Any] | None): - self.data = data or {} - - @classmethod - def read(cls): - return TeleportLocalState(LocalState.read("teleport")) - - def write(self): - LocalState.write("teleport", self.data) - - def get_pid(self, app_name: str) -> str | None: - existing = self.data.get(app_name, {}) - return existing.get("pid") - - def set_pid(self, app_name: str, pid: str | None): - existing = self.data.get(app_name, {}) - existing["pid"] = pid - self.data[app_name] = existing - - def get_address(self, app_name: str) -> str | None: - existing = self.data.get(app_name, {}) - return existing.get("address") - - def set_address(self, app_name: str, addr: str | None): - existing = self.data.get(app_name, {}) - existing["address"] = addr - self.data[app_name] = existing - - def __str__(self): - return dedent(f""" - TeleportLocalState: - data: {self.data} - """) diff --git a/misc/python/materialize/buildkite_insights/segfaults/find.py b/misc/python/materialize/buildkite_insights/segfaults/find.py deleted file mode 100755 index f620474b1993f..0000000000000 --- a/misc/python/materialize/buildkite_insights/segfaults/find.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import time - -from materialize.buildkite_insights.buildkite_api import builds_api, generic_api - - -def main() -> None: - # Used to find recent instances of https://github.com/MaterializeInc/database-issues/issues/7338 - # 2 weeks ~ 2000 builds - data = builds_api.get_builds_of_all_pipelines(max_fetches=20, branch=None) - - for build in data: - request_path = f"organizations/materialize/pipelines/{build['pipeline']['slug']}/builds/{build['number']}/artifacts" - params = {"per_page": "100"} - result = generic_api.get_multiple(request_path, params, max_fetches=None) - for artifact in result: - # Some core files are corrupted, probably because they get dumped during shutdown, ignore them - if ( - "core" in artifact["filename"] - and build["pipeline"]["slug"] != "coverage" - and artifact["file_size"] > 100000 - ): - print( - f"{build['started_at']}: {artifact['filename']} in https://buildkite.com/materialize/{build['pipeline']['slug']}/builds/{build['number']}#{artifact['job_id']}" - ) - time.sleep(2) - - -if __name__ == "__main__": - main() diff --git a/misc/python/materialize/cloudtest/k8s/toxiproxy.py b/misc/python/materialize/cloudtest/k8s/toxiproxy.py index 6dcf93149a63d..a73ee2d17c6da 100644 --- a/misc/python/materialize/cloudtest/k8s/toxiproxy.py +++ b/misc/python/materialize/cloudtest/k8s/toxiproxy.py @@ -21,11 +21,8 @@ V1ServiceSpec, ) -from materialize.cloudtest import DEFAULT_K8S_NAMESPACE from materialize.cloudtest.k8s.api.k8s_deployment import K8sDeployment -from materialize.cloudtest.k8s.api.k8s_resource import K8sResource from materialize.cloudtest.k8s.api.k8s_service import K8sService -from materialize.mzcompose.services.redpanda import REDPANDA_VERSION TOXIPROXY_IMAGE = "jauderho/toxiproxy:v2.8.0" @@ -163,133 +160,3 @@ def delete(self) -> None: self.api().delete_namespaced_service( name=self._name, namespace=self.namespace() ) - - -def toxiproxy_resources( - namespace: str = DEFAULT_K8S_NAMESPACE, - name: str = "toxiproxy", - apply_node_selectors: bool = False, -) -> list[K8sResource]: - """Create Toxiproxy deployment and service resources. - - Args: - namespace: Kubernetes namespace - name: Name for this toxiproxy instance (use different names for multi-AZ) - apply_node_selectors: Whether to apply node selectors - """ - return [ - ToxiproxyDeployment(namespace, name, apply_node_selectors), - ToxiproxyService(namespace, name), - ] - - -class PrivateLinkTestRedpandaDeployment(K8sDeployment): - """Redpanda deployment that advertises an AZ-specific hostname for PrivateLink testing. - - This allows testing pattern-based broker routing where the advertised broker - address contains an AZ identifier that can be matched by routing rules. - - Args: - namespace: Kubernetes namespace - name: Name for this Redpanda instance - advertise_addr: The address Redpanda advertises to clients (e.g., "broker.use1-az1.internal:9092") - apply_node_selectors: Whether to apply node selectors - """ - - def __init__( - self, - namespace: str, - name: str = "redpanda-privatelink", - advertise_addr: str = "broker.use1-az1.internal:9092", - apply_node_selectors: bool = False, - ) -> None: - super().__init__(namespace) - self._name = name - app_label = name - - container = V1Container( - name="redpanda", - image=f"redpandadata/redpanda:{REDPANDA_VERSION}", - command=[ - "/usr/bin/rpk", - "redpanda", - "start", - "--overprovisioned", - "--smp", - "1", - "--memory", - "1G", - "--reserve-memory", - "0M", - "--node-id", - "0", - "--check=false", - "--set", - "redpanda.enable_transactions=true", - "--set", - "redpanda.enable_idempotence=true", - "--set", - "redpanda.auto_create_topics_enabled=true", - "--set", - "redpanda.topic_memory_per_partition=4096", - "--advertise-kafka-addr", - advertise_addr, - ], - ) - - node_selector = None - if apply_node_selectors: - node_selector = {"supporting-services": "true"} - - template = V1PodTemplateSpec( - metadata=V1ObjectMeta(namespace=namespace, labels={"app": app_label}), - spec=V1PodSpec(containers=[container], node_selector=node_selector), - ) - - selector = V1LabelSelector(match_labels={"app": app_label}) - spec = V1DeploymentSpec(replicas=1, template=template, selector=selector) - - self.deployment = V1Deployment( - api_version="apps/v1", - kind="Deployment", - metadata=V1ObjectMeta(name=name, namespace=namespace), - spec=spec, - ) - - def delete(self) -> None: - self.apps_api().delete_namespaced_deployment( - name=self._name, namespace=self.namespace() - ) - - -class PrivateLinkTestRedpandaService(K8sService): - """Service for the PrivateLink test Redpanda instance. - - Args: - namespace: Kubernetes namespace - name: Name for this service (should match deployment name) - """ - - def __init__(self, namespace: str, name: str = "redpanda-privatelink") -> None: - super().__init__(namespace) - self._name = name - app_label = name - - ports = [ - V1ServicePort(name="kafka", port=9092), - V1ServicePort(name="schema-registry", port=8081), - ] - - self.service = V1Service( - metadata=V1ObjectMeta( - name=name, namespace=namespace, labels={"app": app_label} - ), - spec=V1ServiceSpec( - type="NodePort", ports=ports, selector={"app": app_label} - ), - ) - - def delete(self) -> None: - self.api().delete_namespaced_service( - name=self._name, namespace=self.namespace() - ) diff --git a/misc/python/materialize/deb.py b/misc/python/materialize/deb.py deleted file mode 100644 index 88f29035db122..0000000000000 --- a/misc/python/materialize/deb.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -"""Debian packaging utilities.""" diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 18afa831d4b44..7b40c8bc9b23d 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -90,7 +90,6 @@ def get_minimal_system_parameters( "enable_lgalloc": "false", "enable_load_generator_counter": "true", "enable_logical_compaction_window": "true", - "enable_multi_worker_storage_persist_sink": "true", "enable_multi_replica_sources": "true", "enable_rbac_checks": "true", "enable_reduce_mfp_fusion": "true", diff --git a/misc/python/materialize/mzcompose/services/squid.py b/misc/python/materialize/mzcompose/services/squid.py deleted file mode 100644 index 8b3bddc54eb45..0000000000000 --- a/misc/python/materialize/mzcompose/services/squid.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -from materialize.mzcompose.service import ( - Service, -) - - -class Squid(Service): - """ - An HTTP forward proxy, used in some workflows to test whether Materialize can correctly route - traffic via the proxy. - """ - - def __init__( - self, - name: str = "squid", - image: str = "sameersbn/squid:3.5.27-2", - port: int = 3128, - volumes: list[str] = ["./squid.conf:/etc/squid/squid.conf"], - ) -> None: - super().__init__( - name=name, - config={"image": image, "ports": [port], "volumes": volumes}, - ) diff --git a/misc/python/materialize/output_consistency/debug/print_query_result.py b/misc/python/materialize/output_consistency/debug/print_query_result.py deleted file mode 100644 index 075e2c8a2752c..0000000000000 --- a/misc/python/materialize/output_consistency/debug/print_query_result.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -from materialize.output_consistency.query.query_result import ( - QueryResult, -) - - -def _determine_column_lengths( - outcome: QueryResult, min_length: int, max_length: int -) -> list[int]: - column_lengths = [min_length for _ in range(0, outcome.query_column_count)] - - for row_index in range(0, outcome.row_count()): - for col_index in range(0, outcome.query_column_count): - value = outcome.result_rows[row_index][col_index] - value_length = len(str(value)) - column_lengths[col_index] = min( - max(column_lengths[col_index], value_length), max_length - ) - - return column_lengths diff --git a/misc/python/materialize/output_consistency/ignore_filter/param_matchers.py b/misc/python/materialize/output_consistency/ignore_filter/param_matchers.py deleted file mode 100644 index 3014643a7c08e..0000000000000 --- a/misc/python/materialize/output_consistency/ignore_filter/param_matchers.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -from collections.abc import Callable - -from materialize.output_consistency.operation.operation_param import OperationParam - - -def index_of_param( - params: list[OperationParam], match_fn: Callable[[OperationParam], bool] -) -> int | None: - for i, param in enumerate(params): - if match_fn(param): - return i - - return None diff --git a/misc/python/materialize/output_consistency/input_data/params/uuid_operation_param.py b/misc/python/materialize/output_consistency/input_data/params/uuid_operation_param.py deleted file mode 100644 index 9b7d04de34fa6..0000000000000 --- a/misc/python/materialize/output_consistency/input_data/params/uuid_operation_param.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -from materialize.output_consistency.data_type.data_type import DataType -from materialize.output_consistency.data_type.data_type_category import DataTypeCategory -from materialize.output_consistency.expression.expression import Expression -from materialize.output_consistency.expression.expression_characteristics import ( - ExpressionCharacteristics, -) -from materialize.output_consistency.input_data.types.uuid_type_provider import ( - UUID_TYPE_IDENTIFIER, -) -from materialize.output_consistency.operation.operation_param import OperationParam - - -class UuidOperationParam(OperationParam): - def __init__( - self, - optional: bool = False, - incompatibilities: set[ExpressionCharacteristics] | None = None, - ): - super().__init__( - DataTypeCategory.UUID, - optional, - incompatibilities, - incompatibility_combinations=None, - ) - - def supports_type( - self, data_type: DataType, previous_args: list[Expression] - ) -> bool: - return data_type.internal_identifier == UUID_TYPE_IDENTIFIER diff --git a/misc/python/materialize/query_fitness/README.md b/misc/python/materialize/query_fitness/README.md deleted file mode 100644 index 865943b8c65e5..0000000000000 --- a/misc/python/materialize/query_fitness/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains scripts that can be used to pick a smaller subset of interesting queries to test -out of a larger list of queries, such as those produced by a fuzzing tool. diff --git a/misc/python/materialize/query_fitness/all_parts_essential.py b/misc/python/materialize/query_fitness/all_parts_essential.py deleted file mode 100644 index 0c1835f3ef5ea..0000000000000 --- a/misc/python/materialize/query_fitness/all_parts_essential.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -""" -Test that all parts of the query are important. This is done by -commenting out parts of the query -- if any part of the query -can be commented out without this affecting the result of the query -this means that the query contains constructs and predicates that -do not contribute to the final result in any way. - -On the other hand, if all parts of the query are deemed essential, -the query is such that if any part of it is lost during optimization -or execution, the entire query will start producing a different result. -Such queries are suitable for inclusion in regression tests -""" - -from pg8000.dbapi import DatabaseError - -from materialize.query_fitness.fitness_function import FitnessFunction - - -class AllPartsEssential(FitnessFunction): - def _result_checksum(self, query: str) -> str | None: - """Execute the query and return a 'checksum' of the result. - In this implementation, the checksum is simply the serialization of the entire result set - """ - try: - self._cur.execute("COMMIT") - self._cur.execute(query) - return str(self._cur.fetchall()) - except DatabaseError: - return None - - def fitness(self, query: str) -> float: - """Test if all parts of a query are essential to producing the same result. This is done - by commenting out parts of the query and checking if the result is the same. If it is, then - the query contains a non-essential part and is thus rejected (fitness = 0). - """ - query = query.strip(" ;\n") - if not query: - return 0 - - orig_checksum = self._result_checksum(query) - if not orig_checksum: - return 0 - - tokens = query.split() - l = len(tokens) - - for start_token in reversed(range(0, l)): - for end_token in reversed(range(start_token, l)): - new_tokens = [*tokens] - new_tokens.insert(end_token + 1, " */ ") - new_tokens.insert(start_token, " /* ") - - new_query = " ".join(new_tokens) - new_checksum = self._result_checksum(new_query) - - if new_checksum == orig_checksum: - return 0 - - return 1 diff --git a/misc/python/materialize/query_fitness/fitness_function.py b/misc/python/materialize/query_fitness/fitness_function.py deleted file mode 100644 index c9266a2d19bf1..0000000000000 --- a/misc/python/materialize/query_fitness/fitness_function.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -from pg8000 import Connection - - -class FitnessFunction: - def __init__(self, conn: Connection): - self._conn = conn - self._cur = conn.cursor() - - def fitness(self, query: str) -> float: - raise NotImplementedError diff --git a/misc/python/materialize/query_fitness/pick.py b/misc/python/materialize/query_fitness/pick.py deleted file mode 100644 index 0a3628eca12d3..0000000000000 --- a/misc/python/materialize/query_fitness/pick.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -import sys - -import pg8000 -import sqlparse - -from materialize.query_fitness.all_parts_essential import AllPartsEssential - -threshold = 0.5 - - -def main() -> None: - conn = pg8000.connect(user="pstoev", database="pstoev", password="pstoev") - fitness_func = AllPartsEssential(conn=conn) - - for query in sys.stdin: - fitness = fitness_func.fitness(query) - - if fitness > 0.5: - dump_slt(conn, query) - - -def dump_slt(conn: pg8000.Connection, query: str) -> None: - query = sqlparse.format(query.rstrip(), reindent=True, keyword_case="upper") - cursor = conn.cursor() - cursor.execute("ROLLBACK") - cursor.execute(query) - row = cursor.fetchone() - assert row is not None - cols = len(row) - colspec = "I" * cols - print(f""" - -query {colspec} rowsort -{query} ----- -9999999999 values hashing to YY - -query T multiline -EXPLAIN {query} ----- -EOF""") - - -if __name__ == "__main__": - main() diff --git a/misc/python/materialize/setup.py b/misc/python/materialize/setup.py deleted file mode 100644 index 8e42f157a2945..0000000000000 --- a/misc/python/materialize/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License in the LICENSE file at the -# root of this repository, or online at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup - -if __name__ == "__main__": - setup( - name="materialize", - version="0.2.0", - description="Materialize test base.", - author="Materialize, Inc.", - author_email="support@materialize.com", - packages=[ - "materialize", - "materialize.cloudtest", - "materialize.cloudtest.app", - "materialize.cloudtest.k8s", - "materialize.cloudtest.k8s.api", - "materialize.cloudtest.util", - "materialize.mzcompose", - ], - package_dir={"materialize": "."}, - install_requires=["pg8000", "semver", "sqlparse", "kubernetes"], - ) diff --git a/misc/python/materialize/teleport.py b/misc/python/materialize/teleport.py deleted file mode 100644 index dd751adf29c69..0000000000000 --- a/misc/python/materialize/teleport.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import os -import subprocess -import threading -import time -from textwrap import dedent - -import psutil - -from materialize import build_config, ui - - -class TeleportProxy: - @classmethod - def spawn(cls, app_name: str, port: str): - """Spawn a Teleport proxy for the provided app_name.""" - - teleport_state = build_config.TeleportLocalState.read() - - # If there is already a Teleport proxy running, no need to restart one. - running_pid = TeleportProxy.check(app_name) - if running_pid: - ui.say(f"Teleport proxy already running, PID: {running_pid}") - return - else: - # If the existing PID doesn't exist, clear it from state. - teleport_state.set_pid(app_name, None) - teleport_state.set_address(app_name, None) - teleport_state.write() - - # Otherwise spawn a Teleport proxy. - cmd_args = ["tsh", "proxy", "app", f"{app_name}", "--port", port] - child = subprocess.Popen( - cmd_args, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - preexec_fn=os.setpgrp, - ) - ui.say(f"starting Teleport proxy for '{app_name}'...") - - def wait(child, teleport_state, address): - wait_start = time.time() - - while time.time() - wait_start < 2: - child_terminated = child.poll() - if child_terminated: - other_tshs = [ - p.pid - for p in psutil.process_iter(["pid", "name"]) - if p.name() == "tsh" - ] - ui.warn(dedent(f""" - Teleport proxy failed to start, 'tsh' process already running! - existing 'tsh' processes: {other_tshs} - exit code: {child_terminated} - """)) - break - - # Timed out! Check if the process is running. - child_pid_status = psutil.pid_exists(child.pid) - if child_pid_status: - # Record the PID, if the process started successfully. - teleport_state.set_pid(app_name, child.pid) - teleport_state.set_address(app_name, address) - teleport_state.write() - - # Spawn a thread that will wait for the Teleport proxy to start, and - # record it's PID, or warn that it failed to start. - address = f"http://localhost:{port}" - thread = threading.Thread(target=wait, args=[child, teleport_state, address]) - thread.start() - - @classmethod - def check(cls, app_name: str) -> str | None: - """Check if a Teleport proxy is already running for the specified app_name.""" - - teleport_state = build_config.TeleportLocalState.read() - existing_pid = teleport_state.get_pid(app_name) - - if existing_pid and psutil.pid_exists(int(existing_pid)): - return teleport_state.get_pid(app_name) - else: - return None diff --git a/misc/python/materialize/test_analytics/config/sandbox_db_config.py b/misc/python/materialize/test_analytics/config/sandbox_db_config.py deleted file mode 100644 index caae679255ee1..0000000000000 --- a/misc/python/materialize/test_analytics/config/sandbox_db_config.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. diff --git a/misc/python/materialize/test_analytics/setup/test_analytics_setup.py b/misc/python/materialize/test_analytics/setup/test_analytics_setup.py deleted file mode 100644 index c256182181e10..0000000000000 --- a/misc/python/materialize/test_analytics/setup/test_analytics_setup.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import os - -from psycopg import Cursor - -from materialize.test_analytics.util.mz_sql_util import as_sanitized_literal - - -def setup_structures(cursor: Cursor, directory: str) -> None: - if exist_structures(cursor): - return - - setup_files = os.listdir(directory) - setup_files.sort() - - for file_name in setup_files: - if not file_name.endswith(".sql"): - continue - - file_handle = open(f"{directory}/{file_name}") - content = file_handle.read() - - sql_commands = content.split(";") - - for command in sql_commands: - print(f"> {command}") - cursor.execute(command.encode()) - - -def exist_structures(cursor: Cursor) -> bool: - table_name_to_test = "build" - cursor.execute( - f"SELECT exists(SELECT 1 FROM mz_tables WHERE name = {as_sanitized_literal(table_name_to_test)});".encode() - ) - return cursor.fetchall()[0][0] diff --git a/misc/tb/README.md b/misc/tb/README.md deleted file mode 100644 index c9a9750dd8ef6..0000000000000 --- a/misc/tb/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# **t**ail **b**inlogs - -⚠️ **tb is no longer maintained.** ⚠️ - -An archive of the source code is preserved at -https://github.com/MaterializeInc/tb and in the Git history of this repository. diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index e7ab4e8c22546..0000000000000 --- a/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "materialize", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/src/adapter-types/src/compaction.rs b/src/adapter-types/src/compaction.rs index f734bc0f158e5..721160a5b25a4 100644 --- a/src/adapter-types/src/compaction.rs +++ b/src/adapter-types/src/compaction.rs @@ -46,15 +46,6 @@ pub enum CompactionWindow { } impl CompactionWindow { - pub fn lag_from(&self, from: Timestamp) -> Timestamp { - let lag = match self { - CompactionWindow::Default => DEFAULT_LOGICAL_COMPACTION_WINDOW_TS, - CompactionWindow::DisableCompaction => return Timestamp::minimum(), - CompactionWindow::Duration(d) => *d, - }; - from.saturating_sub(lag) - } - /// Returns self as a Timestamp that can be used for comparisons. pub fn comparable_timestamp(&self) -> Timestamp { match self { diff --git a/src/adapter/src/catalog.rs b/src/adapter/src/catalog.rs index c66df7dd95262..34bbae8a04ce4 100644 --- a/src/adapter/src/catalog.rs +++ b/src/adapter/src/catalog.rs @@ -420,43 +420,6 @@ impl Catalog { f(catalog).await } - /// Like [`Catalog::with_debug`], but the catalog created believes that bootstrap is still - /// in progress. - pub async fn with_debug_in_bootstrap(f: F) -> T - where - F: FnOnce(Catalog) -> Fut, - Fut: Future, - { - let persist_client = PersistClient::new_for_tests().await; - let organization_id = Uuid::new_v4(); - let bootstrap_args = test_bootstrap_args(); - let mut catalog = - Self::open_debug_catalog(persist_client.clone(), organization_id, &bootstrap_args) - .await - .expect("can open debug catalog"); - - // Replace `storage` in `catalog` with one that doesn't think bootstrap is over. - let now = SYSTEM_TIME.clone(); - let openable_storage = TestCatalogStateBuilder::new(persist_client) - .with_organization_id(organization_id) - .with_default_deploy_generation() - .build() - .await - .expect("can create durable catalog"); - let mut storage = openable_storage - .open(now().into(), &bootstrap_args) - .await - .expect("can open durable catalog"); - // Drain updates. - let _ = storage - .sync_to_current_updates() - .await - .expect("can sync to current updates"); - catalog.storage = Arc::new(tokio::sync::Mutex::new(storage)); - - f(catalog).await - } - /// Opens a debug catalog. /// /// See [`Catalog::with_debug`]. @@ -797,25 +760,6 @@ impl Catalog { .err_into() } - #[cfg(test)] - pub async fn allocate_system_id( - &self, - commit_ts: mz_repr::Timestamp, - ) -> Result<(CatalogItemId, GlobalId), Error> { - use mz_ore::collections::CollectionExt; - - let mut storage = self.storage().await; - let mut txn = storage.transaction().await?; - let id = txn - .allocate_system_item_ids(1) - .maybe_terminate("allocating system ids")? - .into_element(); - // Drain transaction. - let _ = txn.get_and_commit_op_updates(); - txn.commit(commit_ts).await?; - Ok(id) - } - /// Get the next system item ID without allocating it. pub async fn get_next_system_item_id(&self) -> Result { self.storage() @@ -1014,10 +958,6 @@ impl Catalog { self.state.resolve_builtin_cluster(cluster) } - pub fn get_mz_catalog_server_cluster_id(&self) -> &ClusterId { - &self.resolve_builtin_cluster(&MZ_CATALOG_SERVER_CLUSTER).id - } - /// Resolves a [`Cluster`] for a TargetCluster. pub fn resolve_target_cluster( &self, @@ -1211,16 +1151,6 @@ impl Catalog { } } - pub fn find_available_cluster_name(&self, name: &str) -> String { - let mut i = 0; - let mut candidate = name.to_string(); - while self.state.clusters_by_name.contains_key(&candidate) { - i += 1; - candidate = format!("{}{}", name, i); - } - candidate - } - pub fn get_role_allowed_cluster_sizes(&self, role_id: &Option) -> Vec { if role_id == &Some(MZ_SYSTEM_ROLE_ID) { self.cluster_replica_sizes() diff --git a/src/adapter/src/catalog/state.rs b/src/adapter/src/catalog/state.rs index 15ba6a27bcb39..5502c271b23c7 100644 --- a/src/adapter/src/catalog/state.rs +++ b/src/adapter/src/catalog/state.rs @@ -81,7 +81,7 @@ use mz_sql::plan::{ use mz_sql::rbac; use mz_sql::session::metadata::SessionMetadata; use mz_sql::session::user::MZ_SYSTEM_ROLE_ID; -use mz_sql::session::vars::{DEFAULT_DATABASE_NAME, SystemVars, Var, VarInput}; +use mz_sql::session::vars::{DEFAULT_DATABASE_NAME, SystemVars, VarInput}; use mz_sql_parser::ast::QualifiedReplica; use mz_storage_client::controller::StorageMetadata; use mz_storage_types::connections::ConnectionContext; @@ -1675,11 +1675,6 @@ impl CatalogState { Ok(&cluster.replicas_by_id_[replica_id]) } - /// Get system configuration `name`. - pub fn get_system_configuration(&self, name: &str) -> Result<&dyn Var, Error> { - Ok(self.system_configuration.get(name)?) - } - /// Parse system configuration `name` with `value` int. /// /// Returns the parsed value as a string. diff --git a/src/adapter/src/config/params.rs b/src/adapter/src/config/params.rs index 3972dd3c7c71d..10e5f33aa32b8 100644 --- a/src/adapter/src/config/params.rs +++ b/src/adapter/src/config/params.rs @@ -47,10 +47,6 @@ impl SynchronizedParameters { } } - pub fn is_synchronized(&self, name: &str) -> bool { - self.synchronized.contains(name) - } - /// Return a clone of the set of names of synchronized values. /// /// Mostly useful when we need to iterate over each value, while still diff --git a/src/adapter/src/optimize/dataflows.rs b/src/adapter/src/optimize/dataflows.rs index eb073c1682eab..b6e409e522f72 100644 --- a/src/adapter/src/optimize/dataflows.rs +++ b/src/adapter/src/optimize/dataflows.rs @@ -72,13 +72,6 @@ impl ComputeInstanceSnapshot { }) } - pub fn new_from_parts(instance_id: ComputeInstanceId, collections: BTreeSet) -> Self { - Self { - instance_id, - collections: Some(collections), - } - } - pub fn new_without_collections(instance_id: ComputeInstanceId) -> Self { Self { instance_id, diff --git a/src/adapter/src/session.rs b/src/adapter/src/session.rs index 428e82a1086be..0cf6646975720 100644 --- a/src/adapter/src/session.rs +++ b/src/adapter/src/session.rs @@ -571,14 +571,6 @@ impl Session { Some(notice) } - /// Sets the transaction ops to `TransactionOps::None`. Must only be used after - /// verifying that no transaction anomalies will occur if cleared. - pub fn clear_transaction_ops(&mut self) { - if let Some(txn) = self.transaction.inner_mut() { - txn.ops = TransactionOps::None; - } - } - /// If the current transaction ops belong to a read, then sets the /// ops to `None`, returning the old read timestamp context if /// any existed. Must only be used after verifying that no transaction diff --git a/src/arrow-util/src/builder.rs b/src/arrow-util/src/builder.rs index bdfa9943e175a..f854254c4fff8 100644 --- a/src/arrow-util/src/builder.rs +++ b/src/arrow-util/src/builder.rs @@ -110,21 +110,6 @@ where } impl ArrowBuilder { - /// Helper to validate that a RelationDesc can be encoded into Arrow. - pub fn validate_desc(desc: &RelationDesc) -> Result<(), anyhow::Error> { - let mut errs = vec![]; - for (col_name, col_type) in desc.iter() { - match scalar_to_arrow_datatype(&col_type.scalar_type) { - Ok(_) => {} - Err(_) => errs.push(format!("{}: {:?}", col_name, col_type.scalar_type)), - } - } - if !errs.is_empty() { - anyhow::bail!("Cannot encode the following columns/types: {:?}", errs); - } - Ok(()) - } - /// Helper to validate that a RelationDesc, after applying `overrides`, can /// be encoded into Arrow AND converted from Arrow into parquet by /// arrow-rs's `ArrowWriter`. @@ -271,15 +256,6 @@ impl ArrowBuilder { } } -/// Return the appropriate Arrow DataType for the given SqlScalarType, plus a string -/// that should be used as part of the Arrow 'Extension Type' name for fields using -/// this type: -fn scalar_to_arrow_datatype( - scalar_type: &SqlScalarType, -) -> Result<(DataType, String), anyhow::Error> { - scalar_to_arrow_datatype_impl(scalar_type, &|_| None) -} - /// Returns a description of why this Arrow [`DataType`] cannot be written to /// parquet by arrow-rs's `ArrowWriter`, or `None` if the type is supported. /// Recurses into composite types so a banned type is rejected even if it is diff --git a/src/balancerd/Cargo.toml b/src/balancerd/Cargo.toml index d26bed260feb2..85b63774555ae 100644 --- a/src/balancerd/Cargo.toml +++ b/src/balancerd/Cargo.toml @@ -29,7 +29,7 @@ mz-alloc = { path = "../alloc" } mz-alloc-default = { path = "../alloc-default", optional = true } mz-build-info = { path = "../build-info" } mz-dyncfg-launchdarkly = { path = "../dyncfg-launchdarkly" } -mz-dyncfg-file= { path = "../dyncfg-file" } +mz-dyncfg-file = { path = "../dyncfg-file" } mz-dyncfg = { path = "../dyncfg" } mz-frontegg-auth = { path = "../frontegg-auth" } mz-http-util = { path = "../http-util" } diff --git a/src/catalog/src/durable/transaction.rs b/src/catalog/src/durable/transaction.rs index 5c01ccbe3f80b..c8471aab0291d 100644 --- a/src/catalog/src/durable/transaction.rs +++ b/src/catalog/src/durable/transaction.rs @@ -67,8 +67,8 @@ use crate::durable::{ DATABASE_ID_ALLOC_KEY, DefaultPrivilege, DurableCatalogError, DurableCatalogState, EXPRESSION_CACHE_SHARD_KEY, MOCK_AUTHENTICATION_NONCE_KEY, NetworkPolicy, OID_ALLOC_KEY, SCHEMA_ID_ALLOC_KEY, SYSTEM_CLUSTER_ID_ALLOC_KEY, SYSTEM_ITEM_ALLOC_KEY, - SYSTEM_REPLICA_ID_ALLOC_KEY, Snapshot, SystemConfiguration, USER_ITEM_ALLOC_KEY, - USER_NETWORK_POLICY_ID_ALLOC_KEY, USER_ROLE_ID_ALLOC_KEY, + SYSTEM_REPLICA_ID_ALLOC_KEY, Snapshot, SystemConfiguration, USER_NETWORK_POLICY_ID_ALLOC_KEY, + USER_ROLE_ID_ALLOC_KEY, }; use crate::memory::objects::{StateDiff, StateUpdate, StateUpdateKind}; @@ -350,18 +350,6 @@ impl<'a> Transaction<'a> { Ok((id, oid)) } - pub fn insert_system_schema( - &mut self, - schema_id: u64, - schema_name: &str, - owner_id: RoleId, - privileges: Vec, - oid: u32, - ) -> Result<(), CatalogError> { - let id = SchemaId::System(schema_id); - self.insert_schema(id, None, schema_name.to_string(), owner_id, privileges, oid) - } - pub(crate) fn insert_schema( &mut self, schema_id: SchemaId, @@ -693,44 +681,6 @@ impl<'a> Transaction<'a> { } } - /// Updates persisted information about persisted introspection source - /// indexes. - /// - /// Panics if provided id is not a system id. - pub fn update_introspection_source_index_gids( - &mut self, - mappings: impl Iterator< - Item = ( - ClusterId, - impl Iterator, - ), - >, - ) -> Result<(), CatalogError> { - for (cluster_id, updates) in mappings { - for (name, item_id, index_id, oid) in updates { - let introspection_source_index = IntrospectionSourceIndex { - cluster_id, - name, - item_id, - index_id, - oid, - }; - let (key, value) = introspection_source_index.into_key_value(); - - let prev = self - .introspection_sources - .set(key, Some(value), self.op_id)?; - if prev.is_none() { - return Err(SqlCatalogError::FailedBuiltinSchemaMigration(format!( - "{index_id}" - )) - .into()); - } - } - } - Ok(()) - } - pub fn insert_user_item( &mut self, id: CatalogItemId, @@ -925,18 +875,6 @@ impl<'a> Transaction<'a> { ) } - pub fn allocate_user_item_ids( - &mut self, - amount: u64, - ) -> Result, CatalogError> { - Ok(self - .get_and_increment_id_by(USER_ITEM_ALLOC_KEY.to_string(), amount)? - .into_iter() - // TODO(alter_table): Use separate ID allocators. - .map(|x| (CatalogItemId::User(x), GlobalId::User(x))) - .collect()) - } - pub fn allocate_system_replica_id(&mut self) -> Result { let id = self.get_and_increment_id(SYSTEM_REPLICA_ID_ALLOC_KEY.to_string())?; Ok(ReplicaId::System(id)) @@ -1112,23 +1050,6 @@ impl<'a> Transaction<'a> { } } - /// Removes the database `id` from the transaction. - /// - /// Returns an error if `id` is not found. - /// - /// Runtime is linear with respect to the total number of databases in the catalog. - /// DO NOT call this function in a loop, use [`Self::remove_databases`] instead. - pub fn remove_database(&mut self, id: &DatabaseId) -> Result<(), CatalogError> { - let prev = self - .databases - .set(DatabaseKey { id: *id }, None, self.op_id)?; - if prev.is_some() { - Ok(()) - } else { - Err(SqlCatalogError::UnknownDatabase(id.to_string()).into()) - } - } - /// Removes all databases in `databases` from the transaction. /// /// Returns an error if any id in `databases` is not found. @@ -1158,31 +1079,6 @@ impl<'a> Transaction<'a> { Ok(()) } - /// Removes the schema identified by `database_id` and `schema_id` from the transaction. - /// - /// Returns an error if `(database_id, schema_id)` is not found. - /// - /// Runtime is linear with respect to the total number of schemas in the catalog. - /// DO NOT call this function in a loop, use [`Self::remove_schemas`] instead. - pub fn remove_schema( - &mut self, - database_id: &Option, - schema_id: &SchemaId, - ) -> Result<(), CatalogError> { - let prev = self - .schemas - .set(SchemaKey { id: *schema_id }, None, self.op_id)?; - if prev.is_some() { - Ok(()) - } else { - let database_name = match database_id { - Some(id) => format!("{id}."), - None => "".to_string(), - }; - Err(SqlCatalogError::UnknownSchema(format!("{}.{}", database_name, schema_id)).into()) - } - } - /// Removes all schemas in `schemas` from the transaction. /// /// Returns an error if any id in `schemas` is not found. @@ -1651,40 +1547,6 @@ impl<'a> Transaction<'a> { } } - /// Updates persisted mapping from system objects to global IDs and fingerprints. Each element - /// of `mappings` should be (old-global-id, new-system-object-mapping). - /// - /// Panics if provided id is not a system id. - pub fn update_system_object_mappings( - &mut self, - mappings: BTreeMap, - ) -> Result<(), CatalogError> { - if mappings.is_empty() { - return Ok(()); - } - - let n = self.system_gid_mapping.update( - |_k, v| { - if let Some(mapping) = mappings.get(&CatalogItemId::from(v.catalog_id)) { - let (_, new_value) = mapping.clone().into_key_value(); - Some(new_value) - } else { - None - } - }, - self.op_id, - )?; - - if usize::try_from(n.into_inner()).expect("update diff should fit into usize") - != mappings.len() - { - let id_str = mappings.keys().map(|id| id.to_string()).join(","); - return Err(SqlCatalogError::FailedBuiltinSchemaMigration(id_str).into()); - } - - Ok(()) - } - /// Updates cluster `id` in the transaction to `cluster`. /// /// Returns an error if `id` is not found. @@ -1974,21 +1836,6 @@ impl<'a> Transaction<'a> { Ok(()) } - /// Set persisted replica. - pub fn set_replicas(&mut self, replicas: Vec) -> Result<(), CatalogError> { - if replicas.is_empty() { - return Ok(()); - } - - let replicas = replicas - .into_iter() - .map(DurableType::into_key_value) - .map(|(k, v)| (k, Some(v))) - .collect(); - self.cluster_replicas.set_many(replicas, self.op_id)?; - Ok(()) - } - /// Set persisted configuration. pub fn set_config(&mut self, key: String, value: Option) -> Result<(), CatalogError> { match value { @@ -3706,6 +3553,7 @@ where mod tests { use super::*; + use crate::durable::USER_ITEM_ALLOC_KEY; use mz_controller::clusters::ReplicaLogging; use mz_ore::now::SYSTEM_TIME; use mz_ore::{assert_none, assert_ok}; diff --git a/src/controller/src/lib.rs b/src/controller/src/lib.rs index dbb28dfc3d016..897ee4d123c31 100644 --- a/src/controller/src/lib.rs +++ b/src/controller/src/lib.rs @@ -449,26 +449,6 @@ impl Controller { Ok(ws_id) } - /// Uninstalls a previously installed WatchSetId. The method is a no-op if the watch set has - /// already finished and therefore it's safe to call this function unconditionally. - /// - /// # Panics - /// This method panics if called with a WatchSetId that was never returned by the function. - pub fn uninstall_watch_set(&mut self, ws_id: &WatchSetId) { - if let Some((obj_ids, _)) = self.unfulfilled_watch_sets.remove(ws_id) { - for obj_id in obj_ids { - let mut entry = match self.unfulfilled_watch_sets_by_object.entry(obj_id) { - Entry::Occupied(entry) => entry, - Entry::Vacant(_) => panic!("corrupted watchset state"), - }; - entry.get_mut().remove(ws_id); - if entry.get().is_empty() { - entry.remove(); - } - } - } - } - /// Process a pending response from the storage controller. If necessary, /// return a higher-level response to our client. fn process_storage_response( diff --git a/src/durable-cache/Cargo.toml b/src/durable-cache/Cargo.toml index bc35dc8f7989e..29ed4a78b5ac1 100644 --- a/src/durable-cache/Cargo.toml +++ b/src/durable-cache/Cargo.toml @@ -10,23 +10,13 @@ publish = false workspace = true [dependencies] -async-trait.workspace = true -bytes.workspace = true differential-dataflow.workspace = true -futures.workspace = true -itertools.workspace = true mz-ore = { path = "../ore", features = ["process"] } -mz-dyncfg = { path = "../dyncfg" } mz-persist-types = { path = "../persist-types" } mz-persist-client = { path = "../persist-client" } -mz-timely-util = { path = "../timely-util" } -prometheus.workspace = true -prost.workspace = true -serde = { workspace = true, features = ["rc"] } timely.workspace = true tokio.workspace = true tracing.workspace = true -uuid = { workspace = true, features = ["v4"] } [features] default = [] diff --git a/src/expr/src/relation.rs b/src/expr/src/relation.rs index 202612fe8aa77..1a9e201b4babf 100644 --- a/src/expr/src/relation.rs +++ b/src/expr/src/relation.rs @@ -10,7 +10,7 @@ #![warn(missing_docs)] use std::cell::RefCell; -use std::cmp::{Ordering, max}; +use std::cmp::Ordering; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fmt::{Display, Formatter}; @@ -1563,19 +1563,6 @@ impl MirRelationExpr { std::mem::replace(self, empty) } - /// Replaces `self` with some logic applied to `self`. - pub fn replace_using(&mut self, logic: F) - where - F: FnOnce(MirRelationExpr) -> MirRelationExpr, - { - let empty = MirRelationExpr::Constant { - rows: Ok(vec![]), - typ: ReprRelationType::new(Vec::new()), - }; - let expr = std::mem::replace(self, empty); - *self = logic(expr); - } - /// Store `self` in a `Let` and pass the corresponding `Get` to `body`. pub fn let_in(self, id_gen: &mut IdGen, body: Body) -> Result where @@ -1947,20 +1934,6 @@ impl MirRelationExpr { } } - /// Computes the size (total number of nodes) and maximum depth of a MirRelationExpr for - /// debug printing purposes. - pub fn debug_size_and_depth(&self) -> (usize, usize) { - let mut size = 0; - let mut max_depth = 0; - let mut todo = vec![(self, 1)]; - while let Some((expr, depth)) = todo.pop() { - size += 1; - max_depth = max(max_depth, depth); - todo.extend(expr.children().map(|c| (c, depth + 1))); - } - (size, max_depth) - } - /// The MirRelationExpr is considered potentially expensive if and only if /// at least one of the following conditions is true: /// @@ -2063,59 +2036,6 @@ impl MirRelationExpr { used_across_iterations } - /// Replaces `LetRec` nodes with a stack of `Let` nodes. - /// - /// In each `Let` binding, uses of `Get` in `value` that are not at strictly greater - /// identifiers are rewritten to be the constant collection. - /// This makes the computation perform exactly "one" iteration. - /// - /// This was used only temporarily while developing `LetRec`. - pub fn make_nonrecursive(self: &mut MirRelationExpr) { - let mut deadlist = BTreeSet::new(); - let mut worklist = vec![self]; - while let Some(expr) = worklist.pop() { - if let MirRelationExpr::LetRec { - ids, - values, - limits: _, - body, - } = expr - { - let ids_values = values - .drain(..) - .zip_eq(ids) - .map(|(value, id)| (*id, value)) - .collect::>(); - *expr = body.take_dangerous(); - for (id, mut value) in ids_values.into_iter().rev() { - // Remove references to potentially recursive identifiers. - deadlist.insert(id); - value.visit_pre_mut(|e| { - if let MirRelationExpr::Get { - id: crate::Id::Local(id), - typ, - .. - } = e - { - let typ = typ.clone(); - if deadlist.contains(id) { - e.take_safely(Some(typ)); - } - } - }); - *expr = MirRelationExpr::Let { - id, - value: Box::new(value), - body: Box::new(expr.take_dangerous()), - }; - } - worklist.push(expr); - } else { - worklist.extend(expr.children_mut().rev()); - } - } - } - /// For each Id `id'` referenced in `expr`, if it is larger or equal than `id`, then record in /// `expire_whens` that when `id'` is redefined, then we should expire the information that /// we are holding about `id`. Call `do_expirations` with `expire_whens` at each Id diff --git a/src/http-util/src/lib.rs b/src/http-util/src/lib.rs index eeacb5d3c4593..ee6b18f82cbbd 100644 --- a/src/http-util/src/lib.rs +++ b/src/http-util/src/lib.rs @@ -19,12 +19,10 @@ use axum::response::{Html, IntoResponse, Response}; use axum_extra::TypedHeader; use headers::ContentType; use mz_ore::metrics::MetricsRegistry; -use mz_ore::tracing::TracingHandle; use prometheus::Encoder; use serde::{Deserialize, Serialize}; use serde_json::json; use tower_http::cors::AllowOrigin; -use tracing_subscriber::EnvFilter; /// MIME type used for the Prometheus protobuf scrape format. /// @@ -167,22 +165,6 @@ pub struct DynamicFilterTarget { targets: String, } -/// Dynamically reloads a filter for a tracing layer. -#[allow(clippy::unused_async)] -pub async fn handle_reload_tracing_filter( - handle: &TracingHandle, - reload: fn(&TracingHandle, EnvFilter) -> Result<(), anyhow::Error>, - Json(cfg): Json, -) -> impl IntoResponse { - match cfg.targets.parse::() { - Ok(targets) => match reload(handle, targets) { - Ok(()) => (StatusCode::OK, cfg.targets.to_string()), - Err(e) => (StatusCode::BAD_REQUEST, e.to_string()), - }, - Err(e) => (StatusCode::BAD_REQUEST, e.to_string()), - } -} - /// Returns information about the current status of tracing. #[allow(clippy::unused_async)] pub async fn handle_tracing() -> impl IntoResponse { diff --git a/src/ore/src/bits.rs b/src/ore/src/bits.rs deleted file mode 100644 index bdc4cdbb6b8dd..0000000000000 --- a/src/ore/src/bits.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License in the LICENSE file at the -// root of this repository, or online at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Utilities for bit and byte manipulation - -/// Increases `p` as little as possible (including possibly 0) -/// such that it becomes a multiple of `N`. -pub const fn align_up(p: usize) -> usize { - if p % N == 0 { p } else { p + (N - (p % N)) } -} diff --git a/src/ore/src/graph.rs b/src/ore/src/graph.rs deleted file mode 100644 index d28b8666ffffb..0000000000000 --- a/src/ore/src/graph.rs +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License in the LICENSE file at the -// root of this repository, or online at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Graph utilities. - -use std::collections::BTreeSet; - -/// A non-recursive implementation of a fallible depth-first traversal -/// starting from `root`. -/// -/// Assumes that nodes in the graph all have unique node ids. -/// -/// `at_enter` runs when entering a node. It is expected to return an in-order -/// list of the children of the node. You can omit children from the list -/// returned if you want to skip traversing the subgraphs corresponding to -/// those children. If no children are omitted, `at_enter` can be thought -/// of as a function that processes the nodes of the graph in pre-order. -/// -/// `at_exit` runs when exiting a node. It can be thought of as a function that -/// processes the nodes of the graph in post-order. -/// -/// This function only enters and exits a node at most once and thus is safe to -/// run even if the graph contains a cycle. -pub fn try_nonrecursive_dft( - graph: &Graph, - root: NodeId, - at_enter: &mut AtEnter, - at_exit: &mut AtExit, -) -> Result<(), E> -where - NodeId: std::cmp::Ord, - AtEnter: FnMut(&Graph, &NodeId) -> Result, E>, - AtExit: FnMut(&Graph, &NodeId) -> Result<(), E>, -{ - // All nodes that have been entered but not exited. Last node in the vec is - // the node that we most recently entered. - let mut entered = Vec::new(); - // All nodes that have been exited. - let mut exited = BTreeSet::new(); - - // Pseudocode for the recursive version of this function would look like: - // ``` - // children = at_enter(graph, node) - // foreach child in children: - // recursive_call(graph, child) - // atexit(graph, node) - // ``` - // In this non-recursive implementation, you can think of the call stack as - // been replaced by `entered`. Every time an object is pushed into `entered` - // would have been a time you would have pushed a recursive call onto the - // call stack. Likewise, times an object is popped from `entered` would have - // been times when recursive calls leave the stack. - - // Enter from the root. - let children = at_enter(graph, &root)?; - entered_node(&mut entered, root, children); - while !entered.is_empty() { - if let Some(to_enter) = find_next_child_to_enter(&mut entered, &exited) { - let children = at_enter(graph, &to_enter)?; - entered_node(&mut entered, to_enter, children); - } else { - // If this node has no more children to descend into, - // exit the current node and run `at_exit`. - let (to_exit, _) = entered.pop().unwrap(); - at_exit(graph, &to_exit)?; - exited.insert(to_exit); - } - } - Ok(()) -} - -/// Same as [`try_nonrecursive_dft`], but allows changes to be made to the graph. -pub fn try_nonrecursive_dft_mut( - graph: &mut Graph, - root: NodeId, - at_enter: &mut AtEnter, - at_exit: &mut AtExit, -) -> Result<(), E> -where - NodeId: std::cmp::Ord + Clone, - AtEnter: FnMut(&mut Graph, &NodeId) -> Result, E>, - AtExit: FnMut(&mut Graph, &NodeId) -> Result<(), E>, -{ - // Code in this method is identical to the code in `nonrecursive_dft`. - let mut entered = Vec::new(); - let mut exited = BTreeSet::new(); - - let children = at_enter(graph, &root)?; - entered_node(&mut entered, root, children); - while !entered.is_empty() { - if let Some(to_enter) = find_next_child_to_enter(&mut entered, &exited) { - let children = at_enter(graph, &to_enter)?; - entered_node(&mut entered, to_enter, children); - } else { - let (to_exit, _) = entered.pop().unwrap(); - at_exit(graph, &to_exit)?; - exited.insert(to_exit); - } - } - Ok(()) -} - -/// A non-recursive implementation of an infallible depth-first traversal -/// starting from `root`. -/// -/// Assumes that nodes in the graph all have unique node ids. -/// -/// `at_enter` runs when entering a node. It is expected to return an in-order -/// list of the children of the node. You can omit children from the list -/// returned if you want to skip the traversing subgraphs corresponding to -/// those children. If no children are omitted, `at_enter` can be thought -/// of as a function that processes the nodes of the graph in pre-order. -/// -/// `at_exit` runs when exiting a node. It can be thought of as a function that -/// processes the nodes of the graph in post-order. -/// -/// This function only enters and exits a node at most once and thus is safe to -/// run even if the graph contains a cycle. -pub fn nonrecursive_dft( - graph: &Graph, - root: NodeId, - at_enter: &mut AtEnter, - at_exit: &mut AtExit, -) where - NodeId: std::cmp::Ord, - AtEnter: FnMut(&Graph, &NodeId) -> Vec, - AtExit: FnMut(&Graph, &NodeId) -> (), -{ - // All nodes that have been entered but not exited. Last node in the vec is - // the node that we most recently entered. - let mut entered = Vec::new(); - // All nodes that have been exited. - let mut exited = BTreeSet::new(); - - // Pseudocode for the recursive version of this function would look like: - // ``` - // atenter(graph, node) - // foreach child in children(graph, node): - // recursive_call(graph, child) - // atexit(graph, node) - // ``` - // In this non-recursive implementation, you can think of the call stack as - // been replaced by `entered`. Every time an object is pushed into `entered` - // would have been a time you would have pushed a recursive call onto the - // call stack. Likewise, times an object is popped from `entered` would have - // been times when recursive calls leave the stack. - - // Enter from the root. - let children = at_enter(graph, &root); - entered_node(&mut entered, root, children); - while !entered.is_empty() { - if let Some(to_enter) = find_next_child_to_enter(&mut entered, &exited) { - let children = at_enter(graph, &to_enter); - entered_node(&mut entered, to_enter, children); - } else { - // If this node has no more children to descend into, - // exit the current node and run `at_exit`. - let (to_exit, _) = entered.pop().unwrap(); - at_exit(graph, &to_exit); - exited.insert(to_exit); - } - } -} - -/// Same as [`nonrecursive_dft`], but allows changes to be made to the graph. -pub fn nonrecursive_dft_mut( - graph: &mut Graph, - root: NodeId, - at_enter: &mut AtEnter, - at_exit: &mut AtExit, -) where - NodeId: std::cmp::Ord + Clone, - AtEnter: FnMut(&mut Graph, &NodeId) -> Vec, - AtExit: FnMut(&mut Graph, &NodeId) -> (), -{ - // Code in this method is identical to the code in `nonrecursive_dft`. - let mut entered = Vec::new(); - let mut exited = BTreeSet::new(); - - let children = at_enter(graph, &root); - entered_node(&mut entered, root, children); - while !entered.is_empty() { - if let Some(to_enter) = find_next_child_to_enter(&mut entered, &exited) { - let children = at_enter(graph, &to_enter); - entered_node(&mut entered, to_enter, children); - } else { - let (to_exit, _) = entered.pop().unwrap(); - at_exit(graph, &to_exit); - exited.insert(to_exit); - } - } -} - -/// Add to `entered` that we have entered `node` and `node` has `children`. -fn entered_node( - entered: &mut Vec<(NodeId, Vec)>, - node: NodeId, - mut children: Vec, -) where - NodeId: std::cmp::Ord, -{ - // Reverse children because `find_next_child_to_enter` will traverse the - // list of children by popping them out from the back. - children.reverse(); - entered.push((node, children)) -} - -/// Find the next child node, if any, that we have not entered. -fn find_next_child_to_enter( - entered: &mut Vec<(NodeId, Vec)>, - exited: &BTreeSet, -) -> Option -where - NodeId: std::cmp::Ord, -{ - let (_, children) = entered.last_mut().unwrap(); - while let Some(child) = children.pop() { - if !exited.contains(&child) { - return Some(child); - } - } - None -} diff --git a/src/ore/src/hash.rs b/src/ore/src/hash.rs deleted file mode 100644 index 6d211a3006174..0000000000000 --- a/src/ore/src/hash.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License in the LICENSE file at the -// root of this repository, or online at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Hash utilities. - -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; - -/// Computes the hash of an object implementing [`Hash`]. -pub fn hash(t: &T) -> u64 { - let mut hasher = DefaultHasher::new(); - t.hash(&mut hasher); - hasher.finish() -} diff --git a/src/ore/src/lib.rs b/src/ore/src/lib.rs index fd9b1f40da1f0..b81e04127851d 100644 --- a/src/ore/src/lib.rs +++ b/src/ore/src/lib.rs @@ -25,7 +25,6 @@ #[cfg_attr(nightly_doc_features, doc(cfg(feature = "assert-no-tracing")))] #[cfg(feature = "assert-no-tracing")] pub mod assert; -pub mod bits; #[cfg_attr(nightly_doc_features, doc(cfg(feature = "bytes")))] #[cfg(feature = "bytes")] pub mod bytes; @@ -43,8 +42,6 @@ pub mod fmt; #[cfg_attr(nightly_doc_features, doc(cfg(feature = "async")))] #[cfg(feature = "async")] pub mod future; -pub mod graph; -pub mod hash; pub mod hint; #[cfg(feature = "id_gen")] pub mod id_gen; @@ -70,7 +67,6 @@ pub mod pager; #[cfg(feature = "panic")] pub mod panic; pub mod path; -pub mod permutations; #[cfg_attr(nightly_doc_features, doc(cfg(all(feature = "pool", unix))))] #[cfg(all(feature = "pool", unix))] pub mod pool; diff --git a/src/ore/src/permutations.rs b/src/ore/src/permutations.rs deleted file mode 100644 index 7487166952f2d..0000000000000 --- a/src/ore/src/permutations.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License in the LICENSE file at the -// root of this repository, or online at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Functions for working with permutations - -use std::collections::BTreeMap; - -/// Given a permutation, construct its inverse. -pub fn invert(permutation: I) -> impl Iterator -where - I: IntoIterator, -{ - permutation.into_iter().enumerate().map(|(idx, c)| (c, idx)) -} - -/// Construct the permutation that sorts `data`. -pub fn argsort(data: &[T]) -> Vec -where - T: Ord, -{ - let mut indices = (0..data.len()).collect::>(); - indices.sort_by_key(|&i| &data[i]); - indices -} - -/// Construct the permutation that takes `data.sorted()` to `data`. -pub fn inverse_argsort(data: &[T]) -> Vec -where - T: Ord, -{ - let map = invert(argsort(data)).collect::>(); - (0..data.len()).map(|i| map[&i]).collect() -} diff --git a/src/ore/src/region.rs b/src/ore/src/region.rs index fecad1a44cf3b..114ef031c5819 100644 --- a/src/ore/src/region.rs +++ b/src/ore/src/region.rs @@ -375,22 +375,6 @@ impl Region { pub fn new_heap_zeroed(capacity: usize) -> Self { Self::Heap(vec![T::zeroed(); capacity]) } - - /// Construct a new region with the specified capacity, initialized to 0. - pub fn new_auto_zeroed(capacity: usize) -> Self { - if ENABLE_LGALLOC_REGION.load(std::sync::atomic::Ordering::Relaxed) { - match Region::new_mmap_zeroed(capacity) { - Ok(r) => return r, - Err(lgalloc::AllocError::Disabled) - | Err(lgalloc::AllocError::InvalidSizeClass(_)) => {} - Err(e) => { - eprintln!("lgalloc error: {e}, falling back to heap"); - } - } - } - // Fall-through - Self::new_heap_zeroed(capacity) - } } impl Region { diff --git a/src/persist/src/indexed/encoding.rs b/src/persist/src/indexed/encoding.rs index 70ae0d14bc1e9..13053f4f3f5ca 100644 --- a/src/persist/src/indexed/encoding.rs +++ b/src/persist/src/indexed/encoding.rs @@ -28,9 +28,7 @@ use mz_ore::cast::CastFrom; use mz_ore::collections::CollectionExt; use mz_ore::soft_panic_or_log; use mz_persist_types::arrow::{ArrayBound, ArrayOrd}; -use mz_persist_types::columnar::{ - ColumnEncoder, Schema, codec_to_schema, data_type, schema_to_codec, -}; +use mz_persist_types::columnar::{ColumnEncoder, Schema, codec_to_schema, data_type}; use mz_persist_types::parquet::EncodingConfig; use mz_persist_types::part::Part; use mz_persist_types::schema::backward_compatible; @@ -104,17 +102,6 @@ impl BatchColumnarFormat { BatchColumnarFormat::Both(_) => panic!("unknown batch columnar format"), } } - - /// Returns if we should encode a Batch in a structured format. - pub const fn is_structured(&self) -> bool { - match self { - BatchColumnarFormat::Row => false, - // The V0 format has been deprecated and we ignore its structured columns. - BatchColumnarFormat::Both(0 | 1) => false, - BatchColumnarFormat::Both(_) => true, - BatchColumnarFormat::Structured => true, - } - } } impl fmt::Display for BatchColumnarFormat { @@ -315,33 +302,6 @@ impl BlobTraceUpdates { } } - /// Return the [ColumnarRecords] of the blob, generating it if it does not exist. - pub fn get_or_make_codec( - &mut self, - key_schema: &K::Schema, - val_schema: &V::Schema, - ) -> &ColumnarRecords { - match self { - BlobTraceUpdates::Row(records) => records, - BlobTraceUpdates::Both(records, _) => records, - BlobTraceUpdates::Structured { - key_values, - timestamps, - diffs, - } => { - let key = schema_to_codec::(key_schema, &*key_values.key).expect("valid keys"); - let val = schema_to_codec::(val_schema, &*key_values.val).expect("valid values"); - let records = ColumnarRecords::new(key, val, timestamps.clone(), diffs.clone()); - - *self = BlobTraceUpdates::Both(records, key_values.clone()); - let BlobTraceUpdates::Both(records, _) = self else { - unreachable!("set to BlobTraceUpdates::Both in previous line") - }; - records - } - } - } - /// Return the [`ColumnarRecordsStructuredExt`] of the blob. pub fn get_or_make_structured( &mut self, diff --git a/src/repr/src/adt/datetime.rs b/src/repr/src/adt/datetime.rs index c1bf1a93b3078..d76ac7cccd7df 100644 --- a/src/repr/src/adt/datetime.rs +++ b/src/repr/src/adt/datetime.rs @@ -269,21 +269,6 @@ impl DateTimeField { Interval::convert_date_time_unit(self, Self::Microseconds, 1i64).unwrap() } - - /// Returns the number of months in a single unit of `field`. - /// - /// # Panics - /// - /// Panics if called on a duration field. - pub fn month_multiplier(self) -> i64 { - use DateTimeField::*; - match self { - Millennium | Century | Decade | Year => {} - _other => unreachable!("Do not call with a duration field"), - } - - Interval::convert_date_time_unit(self, Self::Microseconds, 1i64).unwrap() - } } /// An iterator over DateTimeFields diff --git a/src/repr/src/explain/dot.rs b/src/repr/src/explain/dot.rs index 52101398448b4..767130b5bb749 100644 --- a/src/repr/src/explain/dot.rs +++ b/src/repr/src/explain/dot.rs @@ -64,25 +64,3 @@ pub fn dot_string>(t: &T) -> String { DotString::<'_>(t).to_string() } - -/// Apply `f: F` to create a rendering context of type `C` and render the given -/// type `t: T` as [`ExplainFormat::Dot`] within that context. -/// -/// # Panics -/// -/// Panics if the [`DisplayDot::fmt_dot`] call returns a [`fmt::Error`]. -pub fn dot_string_at<'a, T: DisplayDot, C, F: Fn() -> C>(t: &'a T, f: F) -> String { - struct DotStringAt<'a, T, C, F: Fn() -> C> { - t: &'a T, - f: F, - } - - impl, C, F: Fn() -> C> DisplayDot<()> for DotStringAt<'_, T, C, F> { - fn fmt_dot(&self, f: &mut fmt::Formatter<'_>, _ctx: &mut ()) -> fmt::Result { - let mut ctx = (self.f)(); - self.t.fmt_dot(f, &mut ctx) - } - } - - dot_string(&DotStringAt { t, f }) -} diff --git a/src/repr/src/relation.rs b/src/repr/src/relation.rs index 6ca97bf9acb2f..9567add16be81 100644 --- a/src/repr/src/relation.rs +++ b/src/repr/src/relation.rs @@ -1578,18 +1578,6 @@ impl RelationDescBuilder { self } - /// Appends the provided columns to the builder. - pub fn with_columns(mut self, iter: I) -> Self - where - I: IntoIterator, - T: Into, - N: Into, - { - self.columns - .extend(iter.into_iter().map(|(name, ty)| (name.into(), ty.into()))); - self - } - /// Adds a new key for the relation. pub fn with_key(mut self, mut indices: Vec) -> RelationDescBuilder { indices.sort_unstable(); diff --git a/src/repr/src/row.rs b/src/repr/src/row.rs index bfd0e9e4a0e3c..2d27f22fb705a 100644 --- a/src/repr/src/row.rs +++ b/src/repr/src/row.rs @@ -214,19 +214,6 @@ impl Row { self.clone() } - /// Like [`Row::pack`], but the provided iterator is allowed to produce an - /// error, in which case the packing operation is aborted and the error - /// returned. - pub fn try_pack<'a, I, D, E>(iter: I) -> Result - where - I: IntoIterator>, - D: Borrow>, - { - let mut row = Row::default(); - row.packer().try_extend(iter)?; - Ok(row) - } - /// Pack a slice of `Datum`s into a `Row`. /// /// This method has the advantage over `pack` that it can determine the required diff --git a/src/s3-datagen/Cargo.toml b/src/s3-datagen/Cargo.toml deleted file mode 100644 index ed58c6fff0c09..0000000000000 --- a/src/s3-datagen/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "mz-s3-datagen" -description = "Generate S3 test data." -version = "0.0.0" -edition.workspace = true -rust-version.workspace = true -publish = false - -[lints] -workspace = true - -[dependencies] -anyhow.workspace = true -aws-config.workspace = true -aws-sdk-s3.workspace = true -bytefmt.workspace = true -clap.workspace = true -futures.workspace = true -indicatif.workspace = true -mz-aws-util = { path = "../aws-util", features = ["s3"] } -mz-ore = { path = "../ore", features = ["cli"] } -tokio.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true - -[features] -default = [] diff --git a/src/s3-datagen/src/main.rs b/src/s3-datagen/src/main.rs deleted file mode 100644 index 1fb339b340b38..0000000000000 --- a/src/s3-datagen/src/main.rs +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::{io, iter}; - -use aws_sdk_s3::operation::create_bucket::CreateBucketError; -use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration}; -use clap::Parser; -use futures::stream::{self, StreamExt, TryStreamExt}; -use mz_ore::cast::CastFrom; -use mz_ore::cli::{self, CliConfig}; -use mz_ore::error::ErrorExt; -use tracing::{Level, error, event, info}; -use tracing_subscriber::filter::EnvFilter; - -/// Generate meaningless data in S3 to test download speeds -#[derive(Parser)] -struct Args { - /// How large to make each line (record) in Bytes - #[clap(short = 'l', long)] - line_bytes: usize, - - /// How large to make each object, e.g. `1 KiB` - #[clap( - short = 's', - long, - value_parser = parse_object_size, - )] - object_size: usize, - - /// How many objects to create - #[clap(short = 'c', long)] - object_count: usize, - - /// All objects will be inserted into this prefix - #[clap(short = 'p', long)] - key_prefix: String, - - /// All objects will be inserted into this bucket - #[clap(short = 'b', long)] - bucket: String, - - /// Which region to operate in - #[clap(short = 'r', long, default_value = "us-east-1")] - region: String, - - /// Number of copy operations to run concurrently - #[clap(long, default_value = "50")] - concurrent_copies: usize, - - /// Which log messages to emit. - /// - /// See environmentd's `--log-filter` option for details. - #[clap(long, value_name = "FILTER", default_value = "off")] - log_filter: String, -} - -#[tokio::main] -async fn main() { - if let Err(e) = run().await { - error!("{}", e.display_with_causes()); - std::process::exit(1); - } -} - -async fn run() -> anyhow::Result<()> { - let args: Args = cli::parse_args(CliConfig::default()); - - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from(args.log_filter)) - .with_writer(io::stderr) - .init(); - - info!( - "starting up to create {} of data across {} objects in {}/{}", - bytefmt::format(u64::cast_from(args.object_size * args.object_count)), - args.object_count, - args.bucket, - args.key_prefix - ); - - let line = iter::repeat('A') - .take(args.line_bytes) - .chain(iter::once('\n')) - .collect::(); - let mut object_size = 0; - let line_size = line.len(); - let object = iter::repeat(line) - .take_while(|_| { - object_size += line_size; - object_size < args.object_size - }) - .collect::(); - - let config = mz_aws_util::defaults().load().await; - let client = mz_aws_util::s3::new_client(&config); - - let first_object_key = format!("{}{:>05}", args.key_prefix, 0); - - let progressbar = indicatif::ProgressBar::new(u64::cast_from(args.object_count)); - - let bucket_config = match config.region().map(|r| r.as_ref()) { - // us-east-1 is special and is not accepted as a location constraint. - None | Some("us-east-1") => None, - Some(r) => Some( - CreateBucketConfiguration::builder() - .location_constraint(BucketLocationConstraint::from(r)) - .build(), - ), - }; - client - .create_bucket() - .bucket(&args.bucket) - .set_create_bucket_configuration(bucket_config) - .send() - .await - .map(|_| info!("created s3 bucket {}", args.bucket)) - .or_else(|e| match e.into_service_error() { - CreateBucketError::BucketAlreadyOwnedByYou(_) => { - event!(Level::INFO, bucket = %args.bucket, "reusing existing bucket"); - Ok(()) - } - e => Err(e), - })?; - - let mut total_created = 0; - client - .put_object() - .bucket(&args.bucket) - .key(&first_object_key) - .body(object.into_bytes().into()) - .send() - .await?; - total_created += 1; - progressbar.inc(1); - - let copy_source = format!("{}/{}", args.bucket, first_object_key.clone()); - - let copy_reqs = (1..args.object_count).map(|i| { - client - .copy_object() - .bucket(&args.bucket) - .copy_source(©_source) - .key(format!("{}{:>05}", args.key_prefix, i)) - .send() - }); - let mut copy_reqs_stream = stream::iter(copy_reqs).buffer_unordered(args.concurrent_copies); - while let Some(_) = copy_reqs_stream.try_next().await? { - progressbar.inc(1); - total_created += 1; - } - drop(progressbar); - - info!("created {} objects", total_created); - assert_eq!(total_created, args.object_count); - - Ok(()) -} - -fn parse_object_size(s: &str) -> Result { - bytefmt::parse(s).map(usize::cast_from) -} diff --git a/src/sql-server-util/src/inspect.rs b/src/sql-server-util/src/inspect.rs index 8e1edb5468407..14293de14591c 100644 --- a/src/sql-server-util/src/inspect.rs +++ b/src/sql-server-util/src/inspect.rs @@ -169,19 +169,6 @@ fn map_null_lsn_to_retry(result: Result) -> RetryResult -pub async fn increment_lsn(client: &mut Client, lsn: Lsn) -> Result { - static INCREMENT_LSN_QUERY: &str = "SELECT sys.fn_cdc_increment_lsn(@P1);"; - let result = client - .query(INCREMENT_LSN_QUERY, &[&lsn.as_bytes().as_slice()]) - .await?; - - mz_ore::soft_assert_eq_or_log!(result.len(), 1); - parse_lsn(&result[..1]) -} - /// Parse an [`Lsn`] in Decimal(25,0) format of the provided [`tiberius::Row`]. /// /// Returns an error if the provided slice doesn't have exactly one row. diff --git a/src/sql-server-util/src/lib.rs b/src/sql-server-util/src/lib.rs index 8f1f15ddca626..cce94fdb2bac6 100644 --- a/src/sql-server-util/src/lib.rs +++ b/src/sql-server-util/src/lib.rs @@ -314,27 +314,6 @@ impl Client { Ok(()) } - /// Returns the current transaction isolation level for the current session. - pub async fn get_transaction_isolation( - &mut self, - ) -> Result { - const QUERY: &str = "SELECT transaction_isolation_level FROM sys.dm_exec_sessions where session_id = @@SPID;"; - let rows = self.simple_query(QUERY).await?; - match &rows[..] { - [row] => { - let val: i16 = row - .try_get(0) - .context("getting 0th column")? - .ok_or_else(|| anyhow::anyhow!("no 0th column?"))?; - let level = TransactionIsolationLevel::try_from_sql_server(val)?; - Ok(level) - } - other => Err(SqlServerError::InvariantViolated(format!( - "expected one row, got {other:?}" - ))), - } - } - /// Returns the [`EngineEdition`] of the connected instance, querying it on /// first access and caching the result for the life of this [`Client`]. pub async fn engine_edition(&mut self) -> Result { @@ -563,19 +542,6 @@ impl TransactionIsolationLevel { TransactionIsolationLevel::Serializable => "SERIALIZABLE", } } - - /// Try to parse a [`TransactionIsolationLevel`] from the value returned from SQL Server. - fn try_from_sql_server(val: i16) -> Result { - let level = match val { - 1 => TransactionIsolationLevel::ReadUncommitted, - 2 => TransactionIsolationLevel::ReadCommitted, - 3 => TransactionIsolationLevel::RepeatableRead, - 4 => TransactionIsolationLevel::Serializable, - 5 => TransactionIsolationLevel::Snapshot, - x => anyhow::bail!("unknown level {x}"), - }; - Ok(level) - } } #[derive(Derivative)] diff --git a/src/sql/src/plan/statement.rs b/src/sql/src/plan/statement.rs index d69cdac72c989..21c9b32aeaa54 100644 --- a/src/sql/src/plan/statement.rs +++ b/src/sql/src/plan/statement.rs @@ -690,25 +690,6 @@ impl<'a> StatementContext<'a> { }) } - // Creates a `ResolvedItemName::Item` from a `GlobalId` and an - // `UnresolvedItemName`. - pub fn allocate_resolved_item_name( - &self, - id: CatalogItemId, - name: UnresolvedItemName, - ) -> Result { - let partial = normalize::unresolved_item_name(name)?; - let qualified = self.allocate_qualified_name(partial.clone())?; - let full_name = self.allocate_full_name(partial)?; - Ok(ResolvedItemName::Item { - id, - qualifiers: qualified.qualifiers, - full_name, - print_id: true, - version: RelationVersionSelector::Latest, - }) - } - pub fn active_database(&self) -> Option<&DatabaseId> { self.catalog.active_database() } diff --git a/src/sql/src/session/vars/definitions.rs b/src/sql/src/session/vars/definitions.rs index 8bc9d8d9cdddb..2252bfdad4c39 100644 --- a/src/sql/src/session/vars/definitions.rs +++ b/src/sql/src/session/vars/definitions.rs @@ -1909,24 +1909,6 @@ feature_flags!( default: true, enable_for_item_parsing: true, }, - { - name: enable_multi_worker_storage_persist_sink, - desc: "multi-worker storage persist sink", - default: true, - enable_for_item_parsing: true, - }, - { - name: enable_persist_streaming_snapshot_and_fetch, - desc: "use the new streaming consolidate for snapshot_and_fetch", - default: false, - enable_for_item_parsing: true, - }, - { - name: enable_persist_streaming_compaction, - desc: "use the new streaming consolidate for compaction", - default: false, - enable_for_item_parsing: true, - }, { name: enable_raise_statement, desc: "RAISE statement", @@ -2135,12 +2117,6 @@ feature_flags!( enable_for_item_parsing: false, scope: ParameterScope::Cluster, }, - { - name: enable_off_thread_optimization, - desc: "use off-thread optimization in `CREATE` statements", - default: true, - enable_for_item_parsing: false, - }, { name: enable_refresh_every_mvs, desc: "REFRESH EVERY and REFRESH AT materialized views", diff --git a/src/storage/src/statistics.rs b/src/storage/src/statistics.rs index 0e1a93756af51..41a7316c0a04a 100644 --- a/src/storage/src/statistics.rs +++ b/src/storage/src/statistics.rs @@ -691,19 +691,6 @@ impl SourceStatistics { } } - /// Set the `bytes_indexed` to the given value - pub fn set_bytes_indexed(&self, value: i64) { - let mut cur = self.stats.borrow_mut(); - let value = if value < 0 { - tracing::warn!("Unexpected negative value for bytes_indexed {}", value); - 0 - } else { - value.unsigned_abs() - }; - cur.stats.bytes_indexed = Some(value); - cur.prom.bytes_indexed.set(value); - } - /// Update the `records_indexed` stat. /// A positive value will add and a negative value will subtract. pub fn update_records_indexed_by(&self, value: i64) { @@ -728,19 +715,6 @@ impl SourceStatistics { } } - /// Set the `records_indexed` to the given value - pub fn set_records_indexed(&self, value: i64) { - let mut cur = self.stats.borrow_mut(); - let value = if value < 0 { - tracing::warn!("Unexpected negative value for records_indexed {}", value); - 0 - } else { - value.unsigned_abs() - }; - cur.stats.records_indexed = Some(value); - cur.prom.records_indexed.set(value); - } - /// Initialize the `rehydration_latency_ms` stat as `NULL`. pub fn initialize_rehydration_latency_ms(&self) { let mut cur = self.stats.borrow_mut(); diff --git a/src/timely-util/src/columnation.rs b/src/timely-util/src/columnation.rs index 61b5090b6385e..e4db43a2cdd29 100644 --- a/src/timely-util/src/columnation.rs +++ b/src/timely-util/src/columnation.rs @@ -109,24 +109,6 @@ impl ColumnationStack { } } - /// Retain elements that pass a predicate, from a specified offset. - /// - /// This method may or may not reclaim memory in the inner region. - pub fn retain_from bool>(&mut self, index: usize, mut predicate: P) { - let mut write_position = index; - for position in index..self.local.len() { - if predicate(&self[position]) { - self.local.swap(position, write_position); - write_position += 1; - } - } - unsafe { - // Unsafety justified in that `write_position` is no greater than - // `self.local.len()` and so this exposes no invalid data. - self.local.set_len(write_position); - } - } - /// Unsafe access to `local` data. The slices store data that is backed by a region /// allocation. Therefore, it is undefined behavior to mutate elements of the `local` slice. /// @@ -144,17 +126,6 @@ impl ColumnationStack { self.inner.heap_size(callback); } - /// Estimate the consumed memory capacity in bytes, summing both used and total capacity. - #[inline] - pub fn summed_heap_size(&self) -> (usize, usize) { - let (mut length, mut capacity) = (0, 0); - self.heap_size(|len, cap| { - length += len; - capacity += cap - }); - (length, capacity) - } - /// The length in items. #[inline] pub fn len(&self) -> usize { diff --git a/src/transform/src/lib.rs b/src/transform/src/lib.rs index 36f988d6a641d..5cc10e6724762 100644 --- a/src/transform/src/lib.rs +++ b/src/transform/src/lib.rs @@ -86,7 +86,6 @@ pub mod non_null_requirements; pub mod normalize_lets; pub mod normalize_ops; pub mod notice; -pub mod ordering; pub mod predicate_pushdown; pub mod reduce_elision; pub mod reduce_reduction; diff --git a/src/transform/src/ordering.rs b/src/transform/src/ordering.rs deleted file mode 100644 index 88ad92284110d..0000000000000 --- a/src/transform/src/ordering.rs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -//! Transformations that impose a canonical order on the inputs of multi-input -//! relation expressions. diff --git a/src/transform/src/typecheck.rs b/src/transform/src/typecheck.rs index b1833a3857781..2fa12367425ae 100644 --- a/src/transform/src/typecheck.rs +++ b/src/transform/src/typecheck.rs @@ -519,20 +519,6 @@ pub fn column_union( diffs } -/// Returns true when it is safe to treat a `sub` row as an `sup` row -/// -/// In particular, the core types must be equal, and if a column in `sup` is nullable, that column should also be nullable in `sub` -/// Conversely, it is okay to treat a known non-nullable column as nullable: `sub` may be nullable when `sup` is not -pub fn is_subtype_of(sub: &[ReprColumnType], sup: &[ReprColumnType]) -> bool { - if sub.len() != sup.len() { - return false; - } - - sub.iter().zip_eq(sup.iter()).all(|(got, known)| { - (!known.nullable || got.nullable) && got.scalar_type == known.scalar_type - }) -} - /// Characterizes how a Datum differs from a ReprColumnType #[derive(Clone, Debug)] pub enum DatumTypeDifference { diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index 094c20f552554..9d5cf395dcda3 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -263,11 +263,8 @@ enable_notices_for_equals_null enable_notices_for_index_already_exists enable_notices_for_index_empty_key - enable_off_thread_optimization enable_password_auth enable_paused_cluster_readhold_downgrade - enable_persist_streaming_compaction - enable_persist_streaming_snapshot_and_fetch enable_primary_key_not_enforced enable_projection_pushdown_after_relation_cse enable_public_metrics_endpoint @@ -461,6 +458,7 @@ enable_iceberg_sink enable_kafka_sink_partition_by enable_multi_replica_sources + enable_multi_worker_storage_persist_sink enable_reduce_reduction enable_repr_typecheck enable_unified_cluster_arrangment