From e8661dfc7a6973c125adc06cba5ca6ea69d15113 Mon Sep 17 00:00:00 2001 From: Arkesh Mishra <118651144+arkmish@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:20:57 +0530 Subject: [PATCH 1/2] Backport the native snapshot size comparer Adapt Apache ZooKeeper ZOOKEEPER-3427 by Michael Han (f90060b83da4bfcca58ada93a57fedb40a069387) to the existing Java 8 and CLI dependencies. Keep checksummed read-only snapshot loading, native launchers, operator documentation and synthetic CLI tests together. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- bin/zkSnapshotComparer.cmd | 23 + bin/zkSnapshotComparer.sh | 29 ++ .../main/resources/markdown/zookeeperTools.md | 59 +++ .../zookeeper/server/SnapshotComparer.java | 435 ++++++++++++++++++ .../server/SnapshotComparerTest.java | 434 +++++++++++++++++ 5 files changed, 980 insertions(+) create mode 100644 bin/zkSnapshotComparer.cmd create mode 100755 bin/zkSnapshotComparer.sh create mode 100644 zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotComparer.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotComparerTest.java diff --git a/bin/zkSnapshotComparer.cmd b/bin/zkSnapshotComparer.cmd new file mode 100644 index 00000000000..9701d00db7b --- /dev/null +++ b/bin/zkSnapshotComparer.cmd @@ -0,0 +1,23 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal +call "%~dp0zkEnv.cmd" + +set ZOOMAIN=org.apache.zookeeper.server.SnapshotComparer +call %JAVA% -cp "%CLASSPATH%" %ZOOMAIN% %* + +endlocal & exit /b %ERRORLEVEL% diff --git a/bin/zkSnapshotComparer.sh b/bin/zkSnapshotComparer.sh new file mode 100755 index 00000000000..0a2283429f7 --- /dev/null +++ b/bin/zkSnapshotComparer.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ZOOBIN="${BASH_SOURCE-$0}" +ZOOBIN="$(dirname "${ZOOBIN}")" +ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)" + +if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then + . "$ZOOBINDIR"/../libexec/zkEnv.sh +else + . "$ZOOBINDIR"/zkEnv.sh +fi + +"$JAVA" -cp "$CLASSPATH" $JVMFLAGS \ + org.apache.zookeeper.server.SnapshotComparer "$@" diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md b/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md index 94dbd7cfc4e..d5fd1887528 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md @@ -23,6 +23,7 @@ limitations under the License. * [zkCleanup.sh](#zkCleanup) * [zkTxnLogToolkit.sh](#zkTxnLogToolkit) * [zkSnapShotToolkit.sh](#zkSnapShotToolkit) + * [zkSnapshotComparer.sh](#zkSnapshotComparer) * [Testing](#Testing) * [Jepsen Test](#jepsen-test) @@ -212,6 +213,64 @@ USAGE: SnapshotFormatter [-d|-json] snapshot_file [[1,0,{"progname":"SnapshotFormatter.java","progver":"0.01","timestamp":1559788148637},[{"name":"\/","asize":0,"dsize":0,"dev":0,"ino":1001},[{"name":"zookeeper","asize":0,"dsize":0,"dev":0,"ino":1002},{"name":"config","asize":0,"dsize":0,"dev":0,"ino":1003},[{"name":"quota","asize":0,"dsize":0,"dev":0,"ino":1004},[{"name":"test","asize":0,"dsize":0,"dev":0,"ino":1005},{"name":"zookeeper_limits","asize":52,"dsize":52,"dev":0,"ino":1006},{"name":"zookeeper_stats","asize":15,"dsize":15,"dev":0,"ino":1007}]]],{"name":"test","asize":0,"dsize":0,"dev":0,"ino":1008}]] ``` + + +### zkSnapshotComparer.sh + +Compare the subtree data sizes and descendant counts in two snapshot files. +This is the native backport of [ZOOKEEPER-3427](https://github.com/apache/zookeeper/commit/f90060b83da4bfcca58ada93a57fedb40a069387). +The tool reports paths found only in the left or right snapshot and size/count +deltas for paths present in both. Deltas are **right minus left**. + +```bash +bin/zkSnapshotComparer.sh --left snapshot.1 --right snapshot.2.gz --bytes 0 --nodes 0 +``` + +All four options are required: + +* `-l`, `--left`: left snapshot file. +* `-r`, `--right`: right snapshot file. +* `-b`, `--bytes`: non-negative byte-difference threshold. +* `-n`, `--nodes`: non-negative descendant-count-difference threshold. + +Thresholds are integers from 0 through 2147483647. A path is printed when +**either** absolute difference is **strictly greater** than its threshold. +For a path present in only one snapshot, its subtree size and descendant count +are compared to the thresholds. Data sizes include the node's own payload and +all descendant payloads; descendant counts exclude the node itself. Null data +contributes zero bytes. A zero-byte leaf present in only one snapshot is +therefore filtered even with both thresholds set to zero. + +Use `-d`, `--debug` to display filtered paths and comparison details. Use +`-i`, `--interactive` to explore the snapshots interactively: + +* Press Enter to print the current depth and advance. +* Enter a non-negative depth to jump to that layer (the root is depth 0). +* Enter an absolute path, including `/`, to compare its immediate children. +* End input or press Ctrl-C to stop before all layers have been compared. + +The batch report visits each depth and sorts paths alphabetically within it. +It retains the upstream empty label for the root in comparison lines. + +#### Snapshot analysis limitations + +The comparer runs offline, reads files without modifying them, and supports +uncompressed, `.gz`, and `.snappy` snapshots, including mixed formats. It +validates snapshot checksums and reports unreadable or corrupt input rather +than silently producing a successful analysis. Invalid arguments or file paths +exit with code 2; snapshot read failures exit with code 1. A Windows launcher +with a `.cmd` extension is included. Both launchers use the existing `zkEnv` +configuration. + +The comparer **includes ephemeral znodes** present in the snapshot; it does not +report session records. This reflects the upstream traversal behavior, despite +the original description claiming that ephemerals were ignored. It does not +compare payload contents, ACLs, versions or other znode metadata. +**Equal sizes/counts do not prove identical contents.** Snapshots may be fuzzy: +the tool does not replay transaction logs, reconstruct point-in-time state, +or establish transaction-consistent equality. It loads snapshots into memory, +and recursive traversal visits the full snapshot. + ## Testing diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotComparer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotComparer.java new file mode 100644 index 00000000000..39c9616ce27 --- /dev/null +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotComparer.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server; + +import java.io.File; +import java.io.IOException; +import java.io.PushbackInputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.zip.CheckedInputStream; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.jute.BinaryInputArchive; +import org.apache.jute.InputArchive; +import org.apache.zookeeper.ZKUtil; +import org.apache.zookeeper.server.persistence.FileSnap; +import org.apache.zookeeper.server.persistence.SnapStream; +import org.apache.zookeeper.util.ServiceUtils; + +/** + * Compares snapshot subtree data sizes and descendant counts, including ephemeral nodes. + * Sessions, payload contents, ACLs and other metadata are not compared. Equal sizes and + * counts do not establish identical contents or transaction-consistent state. + * + *

Backported from Apache ZooKeeper commit f90060b83da4bfcca58ada93a57fedb40a069387 + * (ZOOKEEPER-3427). + */ +public class SnapshotComparer { + + private static final String LEFT_OPTION = "left"; + private static final String RIGHT_OPTION = "right"; + private static final String BYTE_THRESHOLD_OPTION = "bytes"; + private static final String NODE_THRESHOLD_OPTION = "nodes"; + private static final String DEBUG_OPTION = "debug"; + private static final String INTERACTIVE_OPTION = "interactive"; + + private final Options options = new Options(); + + private SnapshotComparer() { + options.addOption(Option.builder("l").longOpt(LEFT_OPTION).hasArg().required() + .argName("LEFT").desc("(Required) The left snapshot file.").build()); + options.addOption(Option.builder("r").longOpt(RIGHT_OPTION).hasArg().required() + .argName("RIGHT").desc("(Required) The right snapshot file.").build()); + options.addOption(Option.builder("b").longOpt(BYTE_THRESHOLD_OPTION).hasArg().required() + .argName("BYTETHRESHOLD") + .desc("(Required) The node data delta size threshold, in bytes, for printing the node.").build()); + options.addOption(Option.builder("n").longOpt(NODE_THRESHOLD_OPTION).hasArg().required() + .argName("NODETHRESHOLD") + .desc("(Required) The descendant node delta size threshold, in nodes, for printing the node.").build()); + options.addOption("d", DEBUG_OPTION, false, "Use debug output."); + options.addOption("i", INTERACTIVE_OPTION, false, "Enter interactive mode."); + } + + private void usage() { + new HelpFormatter().printHelp(120, "java -cp " + SnapshotComparer.class.getName(), "", options, ""); + } + + public static void main(String[] args) { + new SnapshotComparer().compareSnapshots(args); + } + + private void compareSnapshots(String[] args) { + CommandLine parsedOptions; + File left; + File right; + int byteThreshold; + int nodeThreshold; + try { + parsedOptions = new DefaultParser().parse(options, args); + if (!parsedOptions.getArgList().isEmpty()) { + throw new IllegalArgumentException("Unexpected arguments: " + parsedOptions.getArgList()); + } + byteThreshold = Integer.parseInt(parsedOptions.getOptionValue(BYTE_THRESHOLD_OPTION)); + nodeThreshold = Integer.parseInt(parsedOptions.getOptionValue(NODE_THRESHOLD_OPTION)); + if (byteThreshold < 0 || nodeThreshold < 0) { + throw new IllegalArgumentException("Thresholds must be non-negative integers."); + } + left = new File(parsedOptions.getOptionValue(LEFT_OPTION)); + right = new File(parsedOptions.getOptionValue(RIGHT_OPTION)); + for (File file : new File[]{left, right}) { + String error = ZKUtil.validateFileInput(file.toString()); + if (error != null) { + throw new IllegalArgumentException(error); + } + } + } catch (ParseException | IllegalArgumentException e) { + System.err.println(e.getMessage()); + usage(); + ServiceUtils.requestSystemExit(ExitCode.INVALID_INVOCATION.getValue()); + return; + } + + boolean debug = parsedOptions.hasOption(DEBUG_OPTION); + boolean interactive = parsedOptions.hasOption(INTERACTIVE_OPTION); + System.out.println("Successfully parsed options!"); + try { + TreeInfo leftTree = new TreeInfo(left); + TreeInfo rightTree = new TreeInfo(right); + System.out.println(leftTree); + System.out.println(rightTree); + compareTrees(leftTree, rightTree, byteThreshold, nodeThreshold, debug, interactive); + } catch (IOException e) { + System.err.println("Unable to read snapshot: " + e.getMessage()); + ServiceUtils.requestSystemExit(ExitCode.UNEXPECTED_ERROR.getValue()); + } + } + + private static class TreeInfo { + + private static class TreeNode { + + final String label; + final long size; + final List children = new ArrayList<>(); + long descendantSize; + long descendantCount; + + TreeNode(String label, long size) { + this.label = label; + this.size = size; + } + + void populateChildren(DataTree dataTree, TreeInfo treeInfo, int depth) { + DataNode node = dataTree.getNode(label); + Set childLabels; + synchronized (node) { + childLabels = node.getChildren(); + } + for (String childName : childLabels) { + String childPath = label + "/" + childName; + DataNode childNode = dataTree.getNode(childPath); + long childSize; + synchronized (childNode) { + childSize = childNode.data == null ? 0 : childNode.data.length; + } + TreeNode child = new TreeNode(childPath, childSize); + child.populateChildren(dataTree, treeInfo, depth + 1); + children.add(child); + } + descendantSize = size; + descendantCount = children.size(); + for (TreeNode child : children) { + descendantSize += child.descendantSize; + descendantCount += child.descendantCount; + } + treeInfo.registerNode(this, depth); + } + + } + + final TreeNode root; + long count; + final List> nodesAtDepths = new ArrayList<>(); + final Map nodesByName = new HashMap<>(); + + TreeInfo(File snapshot) throws IOException { + long beginning = System.nanoTime(); + DataTree dataTree = getSnapshot(snapshot); + System.out.printf("Deserialized snapshot in %s in %f seconds%n", + snapshot.getName(), (System.nanoTime() - beginning) / 1_000_000_000.0); + beginning = System.nanoTime(); + DataNode rootNode = dataTree.getNode(""); + long size; + synchronized (rootNode) { + size = rootNode.data == null ? 0 : rootNode.data.length; + } + root = new TreeNode("", size); + root.populateChildren(dataTree, this, 0); + System.out.printf("Processed data tree in %f seconds%n", (System.nanoTime() - beginning) / 1_000_000_000.0); + } + + void registerNode(TreeNode node, int depth) { + while (depth >= nodesAtDepths.size()) { + nodesAtDepths.add(new ArrayList<>()); + } + nodesAtDepths.get(depth).add(node); + nodesByName.put(node.label, node); + count++; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(String.format("Node count: %d%n", count)); + builder.append(String.format("Total size: %d%n", root.descendantSize)); + builder.append(String.format("Max depth: %d%n", nodesAtDepths.size())); + for (int i = 0; i < nodesAtDepths.size(); i++) { + builder.append(String.format("Count of nodes at depth %d: %d%n", i, nodesAtDepths.get(i).size())); + } + return builder.toString(); + } + + } + + static DataTree getSnapshot(File file) throws IOException { + try (CheckedInputStream stream = SnapStream.getInputStream(file); + PushbackInputStream input = new PushbackInputStream(stream)) { + InputArchive archive = BinaryInputArchive.getArchive(input); + DataTree dataTree = new DataTree(); + new FileSnap(null).deserialize(dataTree, new HashMap(), archive); + checkSealIntegrity(stream, archive); + + // Distinguish an absent digest in older snapshots from a truncated digest. + int next = input.read(); + if (next != -1) { + input.unread(next); + dataTree.new ZxidDigest().deserialize(archive); + checkSealIntegrity(stream, archive); + if (input.read() != -1) { + throw new IOException("Unexpected data after snapshot"); + } + } + return dataTree; + } catch (IOException | NumberFormatException e) { + throw new IOException(file + ": " + e, e); + } + } + + private static void checkSealIntegrity(CheckedInputStream stream, InputArchive archive) throws IOException { + // SnapStream's seal checker is package-private on this branch. + long checksum = stream.getChecksum().getValue(); + long expected = archive.readLong("val"); + String path = archive.readString("path"); + if (checksum != expected || !"/".equals(path)) { + throw new IOException("CRC corruption or invalid snapshot seal"); + } + } + + private static void printThresholdInfo(int byteThreshold, int nodeThreshold) { + System.out.printf("Printing analysis for nodes difference larger than %d bytes or node count difference larger than %d.%n", + byteThreshold, nodeThreshold); + } + + private static void compareTrees( + TreeInfo left, + TreeInfo right, + int byteThreshold, + int nodeThreshold, + boolean debug, + boolean interactive) { + int maxDepth = Math.max(left.nodesAtDepths.size(), right.nodesAtDepths.size()); + if (!interactive) { + printThresholdInfo(byteThreshold, nodeThreshold); + for (int i = 0; i < maxDepth; i++) { + System.out.printf("Analysis for depth %d%n", i); + compareLine(left, right, i, byteThreshold, nodeThreshold, debug, false); + } + } else { + try (Scanner scanner = new Scanner(System.in)) { + int currentDepth = 0; + while (currentDepth < maxDepth) { + System.out.printf("Current depth is %d%n", currentDepth); + System.out.println("- Press enter to move to print current depth layer;\n" + + "- Type a number to jump to and print all nodes at a given depth;\n" + + "- Enter an ABSOLUTE path to print the immediate subtree of a node. Path must start with '/'."); + if (!scanner.hasNextLine()) { + System.out.println("End of input."); + return; + } + String input = scanner.nextLine(); + printThresholdInfo(byteThreshold, nodeThreshold); + if (input.isEmpty()) { + System.out.printf("Analysis for depth %d%n", currentDepth); + compareLine(left, right, currentDepth, byteThreshold, nodeThreshold, debug, true); + currentDepth++; + } else if (input.startsWith("/")) { + System.out.printf("Analysis for node %s%n", input); + compareSubtree(left, right, input, byteThreshold, nodeThreshold, debug); + } else { + try { + int depth = Integer.parseInt(input); + if (depth < 0 || depth >= maxDepth) { + System.out.printf("Depth must be in range [%d, %d]%n", 0, maxDepth - 1); + continue; + } + currentDepth = depth; + System.out.printf("Analysis for depth %d%n", currentDepth); + compareLine(left, right, currentDepth, byteThreshold, nodeThreshold, debug, true); + } catch (NumberFormatException e) { + System.out.printf("Input %s is not valid. Depth must be in range [%d, %d]. " + + "Path must be an absolute path which starts with '/'.%n", + input, 0, maxDepth - 1); + } + } + System.out.println(); + } + } + } + System.out.println("All layers compared."); + } + + private static void compareSubtree( + TreeInfo left, + TreeInfo right, + String path, + int byteThreshold, + int nodeThreshold, + boolean debug) { + String label = "/".equals(path) ? "" : path; + TreeInfo.TreeNode leftRoot = left.nodesByName.get(label); + TreeInfo.TreeNode rightRoot = right.nodesByName.get(label); + if (leftRoot == null && rightRoot == null) { + System.out.printf("Path %s is neither found in left tree nor right tree.%n", path); + } else { + List leftList = leftRoot == null ? Collections.emptyList() : leftRoot.children; + List rightList = rightRoot == null ? Collections.emptyList() : rightRoot.children; + compareNodes(leftList, rightList, byteThreshold, nodeThreshold, debug, true); + } + } + + private static void compareLine( + TreeInfo left, + TreeInfo right, + int depth, + int byteThreshold, + int nodeThreshold, + boolean debug, + boolean interactive) { + List leftList = depth >= left.nodesAtDepths.size() + ? Collections.emptyList() : left.nodesAtDepths.get(depth); + List rightList = depth >= right.nodesAtDepths.size() + ? Collections.emptyList() : right.nodesAtDepths.get(depth); + compareNodes(leftList, rightList, byteThreshold, nodeThreshold, debug, interactive); + } + + private static void compareNodes( + List leftList, + List rightList, + int byteThreshold, + int nodeThreshold, + boolean debug, + boolean interactive) { + Comparator comparator = Comparator.comparing(node -> node.label); + Collections.sort(leftList, comparator); + Collections.sort(rightList, comparator); + int leftIndex = 0; + int rightIndex = 0; + while (leftIndex < leftList.size() || rightIndex < rightList.size()) { + TreeInfo.TreeNode leftNode = leftIndex < leftList.size() ? leftList.get(leftIndex) : null; + TreeInfo.TreeNode rightNode = rightIndex < rightList.size() ? rightList.get(rightIndex) : null; + if (leftNode != null && rightNode != null) { + if (debug) { + System.out.printf("Comparing %s to %s%n", leftNode.label, rightNode.label); + } + int result = comparator.compare(leftNode, rightNode); + if (result < 0) { + if (debug) { + System.out.println("left is less"); + } + printOnly(leftNode, "left", byteThreshold, nodeThreshold, debug, interactive); + leftIndex++; + } else if (result > 0) { + if (debug) { + System.out.println("right is less"); + } + printOnly(rightNode, "right", byteThreshold, nodeThreshold, debug, interactive); + rightIndex++; + } else { + if (debug) { + System.out.println("same"); + } + printBoth(leftNode, rightNode, byteThreshold, nodeThreshold, debug, interactive); + leftIndex++; + rightIndex++; + } + } else if (leftNode != null) { + printOnly(leftNode, "left", byteThreshold, nodeThreshold, debug, interactive); + leftIndex++; + } else { + printOnly(rightNode, "right", byteThreshold, nodeThreshold, debug, interactive); + rightIndex++; + } + } + } + + private static void printOnly( + TreeInfo.TreeNode node, + String side, + int byteThreshold, + int nodeThreshold, + boolean debug, + boolean interactive) { + if (node.descendantSize > byteThreshold || node.descendantCount > nodeThreshold) { + System.out.printf("Node %s found only in %s tree. Descendant size: %d. Descendant count: %d%n", + node.label, side, node.descendantSize, node.descendantCount); + } else if (debug || interactive) { + System.out.printf("Filtered %s node %s of size %d%n", side, node.label, node.descendantSize); + } + } + + private static void printBoth( + TreeInfo.TreeNode left, + TreeInfo.TreeNode right, + int byteThreshold, + int nodeThreshold, + boolean debug, + boolean interactive) { + long byteDelta = right.descendantSize - left.descendantSize; + long nodeDelta = right.descendantCount - left.descendantCount; + if (Math.abs(byteDelta) > byteThreshold || Math.abs(nodeDelta) > nodeThreshold) { + System.out.printf("Node %s found in both trees. Delta: %d bytes, %d descendants%n", + left.label, byteDelta, nodeDelta); + } else if (debug || interactive) { + System.out.printf("Filtered node %s of left size %d, right size %d%n", + left.label, left.descendantSize, right.descendantSize); + } + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotComparerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotComparerTest.java new file mode 100644 index 00000000000..f44c75893b2 --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotComparerTest.java @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.commons.io.FileUtils; +import org.apache.jute.BinaryInputArchive; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.server.persistence.FileSnap; +import org.apache.zookeeper.server.persistence.SnapStream; +import org.apache.zookeeper.server.persistence.SnapStream.StreamMode; +import org.apache.zookeeper.test.ClientBase; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +public class SnapshotComparerTest extends ZKTestCase { + + private SnapshotToolTestSupport support; + + @Before + public void setUp() throws IOException { + support = new SnapshotToolTestSupport(); + } + + @After + public void tearDown() throws IOException { + support.close(); + } + + @Test + public void testAddedAndDeletedSubtrees() throws Exception { + DataTree left = new DataTree(); + support.addNode(left, "/app", new byte[1]); + support.addNode(left, "/app/removed", new byte[2]); + support.addNode(left, "/app/removed/leaf", new byte[3]); + DataTree right = new DataTree(); + support.addNode(right, "/app", new byte[1]); + support.addNode(right, "/app/added", new byte[4]); + support.addNode(right, "/app/added/a", new byte[5]); + support.addNode(right, "/app/added/b", null); + + String output = compare(left, right, "0", "0", "-d"); + + assertThat(output, containsString("Node found in both trees. Delta: 4 bytes, 1 descendants")); + assertThat(output, containsString("Node /app found in both trees. Delta: 4 bytes, 1 descendants")); + assertThat(output, containsString("Node /app/removed found only in left tree. Descendant size: 5. Descendant count: 1")); + assertThat(output, containsString("Node /app/removed/leaf found only in left tree. Descendant size: 3. Descendant count: 0")); + assertThat(output, containsString("Node /app/added found only in right tree. Descendant size: 9. Descendant count: 2")); + assertThat(output, containsString("Node /app/added/a found only in right tree. Descendant size: 5. Descendant count: 0")); + assertThat(output, containsString("Filtered right node /app/added/b of size 0")); + assertTrue(output, output.indexOf("Node /app/added found") < output.indexOf("Node /app/removed found")); + assertThat(output, containsString("All layers compared.")); + } + + @Test + public void testSignedByteDeltas() throws Exception { + DataTree left = new DataTree(); + support.addNode(left, "/grow", new byte[1]); + support.addNode(left, "/shrink", new byte[4]); + DataTree right = new DataTree(); + support.addNode(right, "/grow", new byte[4]); + support.addNode(right, "/shrink", new byte[1]); + + String output = compare(left, right, "0", "2147483647"); + + assertThat(output, containsString("Node /grow found in both trees. Delta: 3 bytes, 0 descendants")); + assertThat(output, containsString("Node /shrink found in both trees. Delta: -3 bytes, 0 descendants")); + } + + @Test + public void testByteThresholdIsExclusive() throws Exception { + DataTree left = new DataTree(); + support.addNode(left, "/node", new byte[1]); + DataTree right = new DataTree(); + support.addNode(right, "/node", new byte[5]); + + String filtered = compare(left, right, "4", "0", "--debug"); + String included = compare(left, right, "3", "0"); + + assertThat(filtered, containsString("Filtered node /node of left size 1, right size 5")); + assertThat(filtered, not(containsString("Node /node found"))); + assertThat(included, containsString("Node /node found in both trees. Delta: 4 bytes, 0 descendants")); + } + + @Test + public void testDescendantThresholdWithNullData() throws Exception { + DataTree left = new DataTree(); + support.addNode(left, "/app", null); + DataTree right = new DataTree(); + support.addNode(right, "/app", null); + support.addNode(right, "/app/child", null); + + String filtered = compare(left, right, "2147483647", "1", "-d"); + String included = compare(left, right, "2147483647", "0"); + String removed = compare(right, left, "2147483647", "0"); + + assertThat(filtered, containsString("Filtered node /app of left size 0, right size 0")); + assertThat(filtered, not(containsString("Node /app found"))); + assertThat(included, containsString("Node /app found in both trees. Delta: 0 bytes, 1 descendants")); + assertThat(removed, containsString("Node /app found in both trees. Delta: 0 bytes, -1 descendants")); + } + + @Test + public void testEphemeralNodesAreIncluded() throws Exception { + DataTree right = new DataTree(); + right.createNode("/ephemeral", new byte[7], ZooDefs.Ids.OPEN_ACL_UNSAFE, 0x123L, -1, 1, 1); + + String output = compare(new DataTree(), right, "0", "0"); + + assertThat(output, containsString("Node /ephemeral found only in right tree. Descendant size: 7. Descendant count: 0")); + assertThat(output, containsString("Node found in both trees. Delta: 7 bytes, 1 descendants")); + } + + @Test + public void testEqualLengthPayloadsAreOnlySizeCompared() throws Exception { + DataTree left = new DataTree(); + support.addNode(left, "/equal", "aa".getBytes(StandardCharsets.UTF_8)); + DataTree right = new DataTree(); + support.addNode(right, "/equal", "bb".getBytes(StandardCharsets.UTF_8)); + + String output = compare(left, right, "0", "0", "-d"); + + assertThat(output, containsString("Filtered node /equal of left size 2, right size 2")); + assertThat(output, not(containsString("Node /equal found"))); + assertThat(output, containsString("All layers compared.")); + } + + @Test + public void testMixedCompressionFormats() throws Exception { + DataTree leftTree = new DataTree(); + support.addNode(leftTree, "/node", new byte[1]); + DataTree rightTree = new DataTree(); + support.addNode(rightTree, "/node", new byte[3]); + File left = support.snapshot(leftTree, StreamMode.CHECKED, true); + for (StreamMode mode : StreamMode.values()) { + File right = support.snapshot(rightTree, mode, true); + String output = run("", arguments(left, right, "0", "0")).success(); + assertThat(output, containsString("Node /node found in both trees. Delta: 2 bytes, 0 descendants")); + } + } + + @Test + public void testSnapshotsWithoutDigest() throws Exception { + DataTree rightTree = new DataTree(); + support.addNode(rightTree, "/node", new byte[3]); + File left = support.snapshot(new DataTree(), StreamMode.CHECKED, false); + File right = support.snapshot(rightTree, StreamMode.CHECKED, false); + + String output = run("", arguments(left, right, "0", "0")).success(); + + assertThat(output, containsString("Node /node found only in right tree. Descendant size: 3. Descendant count: 0")); + } + + @Test + public void testInteractivePathsAndDepths() throws Exception { + DataTree leftTree = new DataTree(); + support.addNode(leftTree, "/app", null); + support.addNode(leftTree, "/app/leaf", new byte[1]); + DataTree rightTree = new DataTree(); + support.addNode(rightTree, "/app", null); + support.addNode(rightTree, "/app/leaf", new byte[3]); + File left = support.snapshot(leftTree, StreamMode.CHECKED, true); + File right = support.snapshot(rightTree, StreamMode.CHECKED, true); + + String output = run("/\n/app\n/app/leaf\n/missing\n-1\n999\nbad\n1\n\n\n", + arguments(left, right, "0", "0", "--interactive")).success(); + + assertThat(output, containsString("Analysis for node /")); + assertThat(output, not(containsString("Path / is neither found"))); + assertThat(output, containsString("Node /app found in both trees. Delta: 2 bytes, 0 descendants")); + assertThat(output, containsString("Node /app/leaf found in both trees. Delta: 2 bytes, 0 descendants")); + assertThat(output, not(containsString("Path /app/leaf is neither found"))); + assertThat(output, containsString("Path /missing is neither found in left tree nor right tree.")); + assertThat(output, containsString("Depth must be in range [0, 2]")); + assertThat(output, containsString("Input bad is not valid.")); + assertThat(output, containsString("Analysis for depth 1")); + assertThat(output, containsString("All layers compared.")); + } + + @Test + public void testInteractiveEndOfInput() throws Exception { + File snapshot = support.snapshot(new DataTree(), StreamMode.CHECKED, true); + + String output = run("", arguments(snapshot, snapshot, "0", "0", "-i")).success(); + + assertThat(output, containsString("End of input.")); + assertThat(output, not(containsString("All layers compared."))); + } + + @Test + public void testInvalidArguments() throws Exception { + File snapshot = support.snapshot(new DataTree(), StreamMode.CHECKED, true); + run("").invalidInvocation(); + run("", "-l", snapshot.toString()).invalidInvocation(); + for (String invalid : Arrays.asList("-1", "NaN", "2147483648")) { + run("", arguments(snapshot, snapshot, invalid, "0")).invalidInvocation(); + run("", arguments(snapshot, snapshot, "0", invalid)).invalidInvocation(); + } + run("", arguments(snapshot, snapshot, "0", "0", "--unknown")).invalidInvocation(); + run("", arguments(snapshot, snapshot, "0", "0", "extra")).invalidInvocation(); + } + + @Test + public void testInvalidFiles() throws Exception { + File snapshot = support.snapshot(new DataTree(), StreamMode.CHECKED, true); + File missing = new File(support.directory, "missing"); + for (File invalid : Arrays.asList(missing, support.directory)) { + run("", arguments(invalid, snapshot, "0", "0")).invalidInvocation(); + run("", arguments(snapshot, invalid, "0", "0")).invalidInvocation(); + } + } + + @Test + public void testCorruptSnapshots() throws Exception { + File valid = support.snapshot(new DataTree(), StreamMode.CHECKED, true); + for (File corrupt : support.corruptSnapshots()) { + SnapshotToolTestSupport.Result result = run("", arguments(corrupt, valid, "0", "0")); + result.readFailure(); + assertThat(result.output, not(containsString("All layers compared."))); + } + } + + @Test + public void testShellLauncher() throws Exception { + Assume.assumeFalse(System.getProperty("os.name").startsWith("Windows")); + DataTree rightTree = new DataTree(); + support.addNode(rightTree, "/node", new byte[3]); + File left = support.snapshot(new DataTree(), StreamMode.CHECKED, true); + File right = support.snapshot(rightTree, StreamMode.GZIP, true); + for (String flags : Arrays.asList("", "-Xms32m -Xmx128m")) { + String output = support.runLauncher("zkSnapshotComparer.sh", flags, + "-l", left.toString(), "-r", right.toString(), "-b", "0", "-n", "0").success(); + assertThat(output, containsString("Node /node found only in right tree. Descendant size: 3. Descendant count: 0")); + } + } + + private String compare(DataTree left, DataTree right, String bytes, String nodes, String... flags) throws Exception { + return run("", arguments(support.snapshot(left, StreamMode.CHECKED, true), + support.snapshot(right, StreamMode.CHECKED, true), + bytes, nodes, flags)).success(); + } + + private String[] arguments(File left, File right, String bytes, String nodes, String... flags) { + List args = new ArrayList<>(Arrays.asList("--left", left.toString(), "--right", right.toString(), + "--bytes", bytes, "--nodes", nodes)); + args.addAll(Arrays.asList(flags)); + return args.toArray(new String[0]); + } + + private SnapshotToolTestSupport.Result run(String input, String... args) throws Exception { + return support.run("org.apache.zookeeper.server.SnapshotComparer", input, args); + } + + static class SnapshotToolTestSupport { + + final File directory = ClientBase.createEmptyTestDir(); + private int nextFile; + + SnapshotToolTestSupport() throws IOException { + } + + void addNode(DataTree tree, String path, byte[] data) throws Exception { + tree.createNode(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, 0, -1, 1, 1); + } + + File snapshot(DataTree tree, StreamMode mode, boolean digest) throws IOException { + File snapshots = new File(directory, "snapshots with spaces"); + assertTrue(snapshots.isDirectory() || snapshots.mkdir()); + File file = new File(snapshots, "snapshot." + Integer.toHexString(++nextFile) + mode.getFileExtension()); + StreamMode previousMode = SnapStream.getStreamMode(); + boolean previousDigest = ZooKeeperServer.isDigestEnabled(); + try { + SnapStream.setStreamMode(mode); + ZooKeeperServer.setDigestEnabled(digest); + new FileSnap(null).serialize(tree, Collections.singletonMap(0x123L, 30000), file, false); + } finally { + SnapStream.setStreamMode(previousMode); + ZooKeeperServer.setDigestEnabled(previousDigest); + } + return file; + } + + List corruptSnapshots() throws IOException { + List files = new ArrayList<>(); + File badHeader = snapshot(new DataTree(), StreamMode.CHECKED, true); + byte[] bytes = Files.readAllBytes(badHeader.toPath()); + bytes[0] ^= 1; + Files.write(badHeader.toPath(), bytes); + files.add(badHeader); + + File truncated = snapshot(new DataTree(), StreamMode.CHECKED, true); + bytes = Files.readAllBytes(truncated.toPath()); + Files.write(truncated.toPath(), Arrays.copyOf(bytes, bytes.length / 2)); + files.add(truncated); + + File checksum = snapshot(new DataTree(), StreamMode.CHECKED, true); + bytes = Files.readAllBytes(checksum.toPath()); + ByteArrayInputStream input = new ByteArrayInputStream(bytes); + new FileSnap(null).deserialize(new DataTree(), new HashMap(), + BinaryInputArchive.getArchive(input)); + int sealOffset = bytes.length - input.available(); + bytes[sealOffset] ^= 1; + Files.write(checksum.toPath(), bytes); + files.add(checksum); + + File partialDigest = snapshot(new DataTree(), StreamMode.CHECKED, true); + bytes = Files.readAllBytes(partialDigest.toPath()); + Files.write(partialDigest.toPath(), Arrays.copyOf(bytes, sealOffset + 14)); + files.add(partialDigest); + + File digestSeal = snapshot(new DataTree(), StreamMode.CHECKED, true); + bytes = Files.readAllBytes(digestSeal.toPath()); + bytes[bytes.length - 13] ^= 1; + Files.write(digestSeal.toPath(), bytes); + files.add(digestSeal); + + File gzipTrailer = snapshot(new DataTree(), StreamMode.GZIP, true); + bytes = Files.readAllBytes(gzipTrailer.toPath()); + bytes[bytes.length - 1] ^= 1; + Files.write(gzipTrailer.toPath(), bytes); + files.add(gzipTrailer); + return files; + } + + Result run(String mainClass, String input, String... args) throws Exception { + List command = new ArrayList<>(Arrays.asList( + new File(System.getProperty("java.home"), "bin/java").toString(), + "-Xmx128m", + "-Djava.io.tmpdir=" + System.getProperty("java.io.tmpdir"), + "-cp", + System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")), + mainClass)); + command.addAll(Arrays.asList(args)); + return runProcess(new ProcessBuilder(command), input); + } + + Result runLauncher(String launcher, String flags, String... args) throws Exception { + File script = new File(System.getProperty("basedir", "."), "../bin/" + launcher); + List command = new ArrayList<>(Arrays.asList("bash", script.getCanonicalPath())); + command.addAll(Arrays.asList(args)); + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().put("JAVA_HOME", System.getProperty("java.home")); + builder.environment().put("CLASSPATH", + System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"))); + builder.environment().put("JVMFLAGS", flags); + return runProcess(builder, ""); + } + + private Result runProcess(ProcessBuilder builder, String input) throws Exception { + File outputFile = new File(directory, "output-" + ++nextFile); + File inputFile = new File(directory, "input-" + nextFile); + Files.write(inputFile.toPath(), input.getBytes(StandardCharsets.UTF_8)); + Process process = builder + .redirectInput(inputFile) + .redirectErrorStream(true) + .redirectOutput(outputFile) + .start(); + try { + assertTrue("CLI did not exit: " + builder.command(), process.waitFor(30, TimeUnit.SECONDS)); + return new Result(process.exitValue(), + new String(Files.readAllBytes(outputFile.toPath()), StandardCharsets.UTF_8)); + } finally { + if (process.isAlive()) { + process.destroyForcibly(); + } + } + } + + void close() throws IOException { + FileUtils.deleteDirectory(directory); + } + + static class Result { + + final int exitCode; + final String output; + + Result(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + + String success() { + assertEquals(output, ExitCode.EXECUTION_FINISHED.getValue(), exitCode); + return output; + } + + void invalidInvocation() { + assertEquals(output, ExitCode.INVALID_INVOCATION.getValue(), exitCode); + assertThat(output.toLowerCase(), containsString("usage:")); + } + + void readFailure() { + assertEquals(output, ExitCode.UNEXPECTED_ERROR.getValue(), exitCode); + assertThat(output, containsString("Unable to read snapshot")); + } + } + } + +} From 0570e41029970e26fbb3acbad99658996956547a Mon Sep 17 00:00:00 2001 From: Arkesh Mishra <118651144+arkmish@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:22:15 +0530 Subject: [PATCH 2/2] Backport the recursive snapshot summary tool Adapt Apache ZooKeeper ZOOKEEPER-4566 by Szabolcs Bukros (05b215994f5e145c2758c4089828b57ba471b329). Reuse the comparer snapshot loader and fixtures, retaining subtree totals, depth behavior, launchers and CLI regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- bin/zkSnapshotRecursiveSummaryToolkit.cmd | 23 ++ bin/zkSnapshotRecursiveSummaryToolkit.sh | 29 +++ .../main/resources/markdown/zookeeperTools.md | 61 ++++-- .../server/SnapshotRecursiveSummary.java | 121 +++++++++++ .../server/SnapshotRecursiveSummaryTest.java | 198 ++++++++++++++++++ 5 files changed, 419 insertions(+), 13 deletions(-) create mode 100644 bin/zkSnapshotRecursiveSummaryToolkit.cmd create mode 100755 bin/zkSnapshotRecursiveSummaryToolkit.sh create mode 100644 zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotRecursiveSummaryTest.java diff --git a/bin/zkSnapshotRecursiveSummaryToolkit.cmd b/bin/zkSnapshotRecursiveSummaryToolkit.cmd new file mode 100644 index 00000000000..850704a3719 --- /dev/null +++ b/bin/zkSnapshotRecursiveSummaryToolkit.cmd @@ -0,0 +1,23 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal +call "%~dp0zkEnv.cmd" + +set ZOOMAIN=org.apache.zookeeper.server.SnapshotRecursiveSummary +call %JAVA% -cp "%CLASSPATH%" %ZOOMAIN% %* + +endlocal & exit /b %ERRORLEVEL% diff --git a/bin/zkSnapshotRecursiveSummaryToolkit.sh b/bin/zkSnapshotRecursiveSummaryToolkit.sh new file mode 100755 index 00000000000..d43be95a84c --- /dev/null +++ b/bin/zkSnapshotRecursiveSummaryToolkit.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ZOOBIN="${BASH_SOURCE-$0}" +ZOOBIN="$(dirname "${ZOOBIN}")" +ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)" + +if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then + . "$ZOOBINDIR"/../libexec/zkEnv.sh +else + . "$ZOOBINDIR"/zkEnv.sh +fi + +"$JAVA" -cp "$CLASSPATH" $JVMFLAGS \ + org.apache.zookeeper.server.SnapshotRecursiveSummary "$@" diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md b/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md index d5fd1887528..7fc685d9e8f 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md @@ -24,6 +24,7 @@ limitations under the License. * [zkTxnLogToolkit.sh](#zkTxnLogToolkit) * [zkSnapShotToolkit.sh](#zkSnapShotToolkit) * [zkSnapshotComparer.sh](#zkSnapshotComparer) + * [zkSnapshotRecursiveSummaryToolkit.sh](#zkSnapshotRecursiveSummaryToolkit) * [Testing](#Testing) * [Jepsen Test](#jepsen-test) @@ -252,24 +253,58 @@ Use `-d`, `--debug` to display filtered paths and comparison details. Use The batch report visits each depth and sorts paths alphabetically within it. It retains the upstream empty label for the root in comparison lines. + + +### zkSnapshotRecursiveSummaryToolkit.sh + +Recursively summarize one snapshot subtree. This is the native backport of +[ZOOKEEPER-4566](https://github.com/apache/zookeeper/commit/05b215994f5e145c2758c4089828b57ba471b329). + +```bash +bin/zkSnapshotRecursiveSummaryToolkit.sh snapshot.1 /app 1 +``` + +Usage: `SnapshotRecursiveSummary `. +The starting node must be an existing absolute znode path. The maximum depth is +a non-negative integer: 0 prints every non-leaf node, 1 prints the starting +node and its non-leaf children, 2 adds another level, and so on. +**Depth limits output only, not traversal or totals.** + +`children` is the number of all descendants, not just immediate children. +`data` is the sum of payload bytes for the node itself and all descendants. +Null data contributes zero bytes. Leaves contribute to their ancestors' totals +but are not printed; selecting a leaf produces no summary entries. + +For example, if `/app` has 2 payload bytes, `/app/branch` has 3, and +`/app/branch/leaf` has 5, the summary is: + +```text + /app + children: 2 + data: 10 +-- /app/branch +-- children: 1 +-- data: 8 +``` + #### Snapshot analysis limitations -The comparer runs offline, reads files without modifying them, and supports -uncompressed, `.gz`, and `.snappy` snapshots, including mixed formats. It -validates snapshot checksums and reports unreadable or corrupt input rather -than silently producing a successful analysis. Invalid arguments or file paths -exit with code 2; snapshot read failures exit with code 1. A Windows launcher -with a `.cmd` extension is included. Both launchers use the existing `zkEnv` -configuration. +Both tools run offline, read files without modifying them, and support +uncompressed, `.gz`, and `.snappy` snapshots (including mixed formats in the +comparer). They validate snapshot checksums and report unreadable or corrupt +input rather than silently producing a successful analysis. Invalid arguments, +file paths or summary starting paths exit with code 2; snapshot read failures +exit with code 1. Windows launchers with the same names and a `.cmd` extension +are also included. The launchers use the existing `zkEnv` configuration. -The comparer **includes ephemeral znodes** present in the snapshot; it does not +Both tools **include ephemeral znodes** present in the snapshot; they do not report session records. This reflects the upstream traversal behavior, despite -the original description claiming that ephemerals were ignored. It does not -compare payload contents, ACLs, versions or other znode metadata. +the original comparer's description claiming that ephemerals were ignored. +Neither tool compares payload contents, ACLs, versions or other znode metadata. **Equal sizes/counts do not prove identical contents.** Snapshots may be fuzzy: -the tool does not replay transaction logs, reconstruct point-in-time state, -or establish transaction-consistent equality. It loads snapshots into memory, -and recursive traversal visits the full snapshot. +these tools do not replay transaction logs, reconstruct point-in-time state, +or establish transaction-consistent equality. The tools load snapshots into +memory, and recursive traversal still visits the full selected subtree. diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java new file mode 100644 index 00000000000..d34bc38c4b6 --- /dev/null +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.zookeeper.ZKUtil; +import org.apache.zookeeper.common.PathUtils; +import org.apache.zookeeper.util.ServiceUtils; + +/** + * Recursively summarizes snapshot subtree data sizes and descendant counts. + * Only non-leaf nodes are printed, but totals include all descendants and ephemeral nodes. + * The maximum depth limits output, not traversal; zero means unlimited output depth. + * + *

Backported from Apache ZooKeeper commit 05b215994f5e145c2758c4089828b57ba471b329 + * (ZOOKEEPER-4566). + */ +@InterfaceAudience.Public +public class SnapshotRecursiveSummary { + + public static void main(String[] args) { + if (args.length != 3) { + System.err.println(getUsage()); + ServiceUtils.requestSystemExit(ExitCode.INVALID_INVOCATION.getValue()); + return; + } + try { + new SnapshotRecursiveSummary().run(args[0], args[1], Integer.parseInt(args[2])); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.err.println(getUsage()); + ServiceUtils.requestSystemExit(ExitCode.INVALID_INVOCATION.getValue()); + } catch (IOException e) { + System.err.println("Unable to read snapshot: " + e.getMessage()); + ServiceUtils.requestSystemExit(ExitCode.UNEXPECTED_ERROR.getValue()); + } + } + + public void run(String snapshotFileName, String startingNode, int maxDepth) throws IOException { + PathUtils.validatePath(startingNode); + if (maxDepth < 0) { + throw new IllegalArgumentException("max_depth must be a non-negative integer."); + } + String error = ZKUtil.validateFileInput(snapshotFileName); + if (error != null) { + throw new IllegalArgumentException(error); + } + DataTree dataTree = SnapshotComparer.getSnapshot(new File(snapshotFileName)); + if (dataTree.getNode(startingNode) == null) { + throw new IllegalArgumentException("Starting node does not exist: " + startingNode); + } + StringBuilder builder = new StringBuilder(); + printZnode(dataTree, startingNode, builder, 0, maxDepth); + System.out.println(builder); + } + + private long[] printZnode(DataTree dataTree, String name, StringBuilder builder, int level, int maxDepth) { + DataNode node = dataTree.getNode(name); + Set children; + long dataSize; + synchronized (node) { + dataSize = node.data == null ? 0 : node.data.length; + children = new TreeSet<>(node.getChildren()); + } + long[] result = {1L, dataSize}; + if (children.isEmpty()) { + return result; + } + StringBuilder childBuilder = new StringBuilder(); + for (String child : children) { + long[] childResult = printZnode(dataTree, name + (name.equals("/") ? "" : "/") + child, + childBuilder, level + 1, maxDepth); + result[0] += childResult[0]; + result[1] += childResult[1]; + } + if (maxDepth == 0 || level <= maxDepth) { + String indent = String.join("", Collections.nCopies(level, "--")); + builder.append(indent).append(" ").append(name).append("\n"); + builder.append(indent).append(" children: ").append(result[0] - 1).append("\n"); + builder.append(indent).append(" data: ").append(result[1]).append("\n"); + builder.append(childBuilder); + } + return result; + } + + public static String getUsage() { + String newLine = System.lineSeparator(); + return String.join(newLine, + "USAGE:", + "", + "SnapshotRecursiveSummary ", + "", + "snapshot_file: path to the zookeeper snapshot", + "starting_node: the absolute path in the zookeeper tree where traversal should begin", + "max_depth: non-negative output depth. 0 displays every non-leaf node; " + + "1 displays the starting node and its non-leaf children; 2 adds another level, and so on. " + + "This ONLY affects the level of details displayed, NOT the calculation."); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotRecursiveSummaryTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotRecursiveSummaryTest.java new file mode 100644 index 00000000000..2e326ee1fc6 --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/SnapshotRecursiveSummaryTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.server.SnapshotComparerTest.SnapshotToolTestSupport; +import org.apache.zookeeper.server.persistence.SnapStream.StreamMode; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +public class SnapshotRecursiveSummaryTest extends ZKTestCase { + + private SnapshotToolTestSupport support; + + @Before + public void setUp() throws IOException { + support = new SnapshotToolTestSupport(); + } + + @After + public void tearDown() throws IOException { + support.close(); + } + + @Test + public void testRecursiveSubtreeTotals() throws Exception { + String output = summarize(tree(), "/app", "0"); + + assertThat(output, containsString(" /app\n children: 5\n data: 17")); + assertThat(output, containsString("-- /app/branch\n-- children: 2\n-- data: 8")); + assertThat(output, containsString("---- /app/branch/deep\n---- children: 1\n---- data: 5")); + assertThat(output, not(containsString(" /app/leaf\n"))); + assertThat(output, not(containsString(" /app/empty\n"))); + assertThat(output, not(containsString(" /zookeeper\n"))); + } + + @Test + public void testDepthOnlyLimitsOutput() throws Exception { + String output = summarize(tree(), "/app", "1"); + + assertThat(output, containsString(" /app\n children: 5\n data: 17")); + assertThat(output, containsString("-- /app/branch\n-- children: 2\n-- data: 8")); + assertThat(output, not(containsString(" /app/branch/deep\n"))); + } + + @Test + public void testRootIncludesItsOwnData() throws Exception { + DataTree tree = tree(); + tree.setData("/", new byte[11], 1, 1, 1); + + String output = summarize(tree, "/", "1"); + + assertThat(output, containsString(" /\n children: 9\n data: 28")); + assertThat(output, containsString("-- /app\n-- children: 5\n-- data: 17")); + assertThat(output, containsString("-- /zookeeper\n-- children: 2\n-- data: 0")); + } + + @Test + public void testSelectedLeafIsNotPrinted() throws Exception { + String output = summarize(tree(), "/app/leaf", "0"); + + assertThat(output, not(containsString("children:"))); + assertThat(output, not(containsString("data:"))); + } + + @Test + public void testNullDataContributesZeroBytes() throws Exception { + DataTree tree = new DataTree(); + support.addNode(tree, "/app", null); + support.addNode(tree, "/app/leaf", null); + + String output = summarize(tree, "/app", "0"); + + assertThat(output, containsString(" /app\n children: 1\n data: 0")); + } + + @Test + public void testEphemeralDescendantsAreIncluded() throws Exception { + DataTree tree = new DataTree(); + support.addNode(tree, "/app", new byte[2]); + tree.createNode("/app/ephemeral", new byte[7], ZooDefs.Ids.OPEN_ACL_UNSAFE, 0x123L, -1, 1, 1); + + String output = summarize(tree, "/app", "0"); + + assertThat(output, containsString(" /app\n children: 1\n data: 9")); + } + + @Test + public void testCompressedSnapshots() throws Exception { + for (StreamMode mode : StreamMode.values()) { + File snapshot = support.snapshot(tree(), mode, true); + String output = run(snapshot.toString(), "/app", "0").success(); + assertThat(output, containsString(" /app\n children: 5\n data: 17")); + } + } + + @Test + public void testSnapshotWithoutDigest() throws Exception { + File snapshot = support.snapshot(tree(), StreamMode.CHECKED, false); + + String output = run(snapshot.toString(), "/app", "0").success(); + + assertThat(output, containsString(" /app\n children: 5\n data: 17")); + } + + @Test + public void testInvalidArguments() throws Exception { + File snapshot = support.snapshot(tree(), StreamMode.CHECKED, true); + run().invalidInvocation(); + run(snapshot.toString(), "/app").invalidInvocation(); + run(snapshot.toString(), "/app", "0", "extra").invalidInvocation(); + for (String invalid : Arrays.asList("-1", "NaN", "2147483648")) { + run(snapshot.toString(), "/app", invalid).invalidInvocation(); + } + } + + @Test + public void testInvalidZnodePaths() throws Exception { + File snapshot = support.snapshot(tree(), StreamMode.CHECKED, true); + for (String invalid : Arrays.asList("", "app", "/app/", "/app//leaf", "/app/..", "/missing")) { + SnapshotToolTestSupport.Result result = run(snapshot.toString(), invalid, "0"); + result.invalidInvocation(); + assertThat(result.output, not(containsString("NullPointerException"))); + } + } + + @Test + public void testInvalidFiles() throws Exception { + run(new File(support.directory, "missing").toString(), "/", "0").invalidInvocation(); + run(support.directory.toString(), "/", "0").invalidInvocation(); + } + + @Test + public void testCorruptSnapshots() throws Exception { + for (File corrupt : support.corruptSnapshots()) { + SnapshotToolTestSupport.Result result = run(corrupt.toString(), "/", "0"); + result.readFailure(); + assertThat(result.output, not(containsString("children:"))); + } + } + + @Test + public void testShellLauncher() throws Exception { + Assume.assumeFalse(System.getProperty("os.name").startsWith("Windows")); + File snapshot = support.snapshot(tree(), StreamMode.GZIP, true); + for (String flags : Arrays.asList("", "-Xms32m -Xmx128m")) { + String output = support.runLauncher("zkSnapshotRecursiveSummaryToolkit.sh", flags, + snapshot.toString(), "/app", "0").success(); + assertThat(output, containsString(" /app\n children: 5\n data: 17")); + } + } + + private DataTree tree() throws Exception { + DataTree tree = new DataTree(); + support.addNode(tree, "/app", new byte[2]); + support.addNode(tree, "/app/branch", new byte[3]); + support.addNode(tree, "/app/branch/deep", null); + support.addNode(tree, "/app/branch/deep/leaf", new byte[5]); + support.addNode(tree, "/app/empty", null); + support.addNode(tree, "/app/leaf", new byte[7]); + return tree; + } + + private String summarize(DataTree tree, String path, String depth) throws Exception { + File snapshot = support.snapshot(tree, StreamMode.CHECKED, true); + return run(snapshot.toString(), path, depth).success(); + } + + private SnapshotToolTestSupport.Result run(String... args) throws Exception { + return support.run("org.apache.zookeeper.server.SnapshotRecursiveSummary", "", args); + } + +}