diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..238988c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.py] +indent_style = space +indent_size = 4 + +[*.{yml,yaml,json,toml}] +indent_style = space +indent_size = 2 + +[Makefile] +indent_style = tab diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..93f5e41 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +* text=auto +*.py text eol=lf +*.md text eol=lf +*.toml text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.sh text eol=lf +*.png binary +*.jpg binary +*.tflite binary diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..5feaa8e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,22 @@ +--- +name: Bug report +about: Report a reproducible ImageDT behavior problem +labels: bug +--- + +## Expected and actual behavior + +## Minimal reproduction + +Use synthetic input and temporary files. Do not include credentials or private data. + +## Environment + +- ImageDT version: +- Python version: +- Operating system: +- Installed optional extras/backends and versions: + +## Error output + +Paste the relevant exception and traceback after removing sensitive paths/data. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0086358 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..b75b6cc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,13 @@ +--- +name: Feature request +about: Propose a concrete utility or compatibility improvement +labels: enhancement +--- + +## Problem or use case + +## Proposed behavior + +## Existing workaround and compatibility constraints + +Include the desired public interface, input/output example and affected backend if relevant. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a6c67fc --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,17 @@ +## Problem and change + +What failed or was missing, and what happens after this change? + +## Compatibility + +Mention changed public names, argument names, output shapes/numerical semantics, dependencies or checkpoint behavior. Write “unchanged” when applicable. + +## Validation + +List commands run and observed pass/skip counts. Name backends/platforms that were not tested. + +- [ ] Ruff lint and format checks pass +- [ ] Relevant behavior and failure/boundary tests pass +- [ ] Legacy API fixture remains valid +- [ ] User-facing docs and changelog updated where needed +- [ ] No credentials, private data or built distributions committed diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a4b0eed --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,41 @@ +name: Python package checks +on: + push: + branches: [main, master, 'codex/**'] + pull_request: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + core: + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.12', '3.14'] + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: python -m pip install ".[dev,plot,excel,pdf,caffe]" + - run: python -m ruff check imagedt tests scripts test + - run: python -m ruff format --check imagedt tests scripts test setup.py + - run: python -m pytest -q + - run: python -m build + - run: python -m twine check dist/* + - run: python scripts/check_release.py + tensorflow: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install ".[dev,plot,excel,pdf,tensorflow]" + - run: python -m pytest --cov=imagedt --cov-report=term-missing --cov-fail-under=88 diff --git a/.gitignore b/.gitignore index 3b751bb..19d4a5f 100644 --- a/.gitignore +++ b/.gitignore @@ -108,4 +108,16 @@ venv.bak/ *.tfrecords *.ipynb *.xml -test/sources/ \ No newline at end of file +test/sources/ +# Python 3 refactor build and test artifacts +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ +coverage.xml +build/ +dist/ +*.egg-info/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bc49df6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +## 0.6.0rc1 — 2026-10-12 + +Python 3 migration candidate. Requires Python 3.10+ and retains historical public names, module paths and argument names. + +- Add modern package metadata, optional dependency groups, portable PDF rendering, CI and isolated release builds. +- Make imports lazy and remove global GPU/eager-mode changes. +- Fix image cleanup, padding, hashing, XML/CSV I/O, filename handling and paired rename failures. +- Prevent annotation overwrites during MD5 pair renaming; preserve split-specific corrupt-image reports and use zero-based converter class IDs. +- Correct BatchNorm protobuf normalization and terminal Caffe output aliases; reject numerically non-equivalent padding. +- Add contributor/testing/security guidance, editor conventions, strict Ruff/format CI checks and PR/issue templates. +- Correct detection/classification F1 and train/validation record leakage; accelerate ROC and Caffe DAG ordering. +- Migrate TensorFlow graph execution, Caffe protobuf, LeNet and Lite inference to supported Python 3 backends. +- Modernize synthetic text, manage pools/sessions/figures, and add self-contained regression, API compatibility and backend integration tests. +- Retain removed CoreML/handwriting/legacy TOCO entrances with explicit unsupported errors. + +Read `docs/migration.md` for deliberate behavioral changes and checkpoint compatibility limits. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a4549ca --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,47 @@ +# Contributing to ImageDT + +ImageDT is an Apache-2.0 Python utility package. Bug reports, regression fixtures and focused pull requests are welcome. Contributions use the existing project license; preserve copyright and license notices when modifying vendored code. + +## Set up a development environment + +Use Python 3.10 or newer. Create an isolated environment and install the project with the capabilities you will change: + +```bash +python3 -m venv .venv +# Linux/macOS: source .venv/bin/activate +# Windows PowerShell: .venv\Scripts\Activate.ps1 +python -m pip install -e '.[dev,plot,excel,pdf,caffe]' +# For model conversion / TensorFlow integration: +python -m pip install -e '.[dev,plot,excel,pdf,tensorflow]' +``` + +Work on a feature branch based on `main`. Keep each commit focused and pair behavior changes with tests. Use commit subjects such as `fix: handle empty annotations` or `docs: clarify optional backends`. A PR should explain the problem, changed behavior, checks executed, and any compatibility limits. Do not commit generated distributions, virtual environments, credentials, personal datasets or machine-specific paths. + +## Python code conventions + +- Follow [PEP 8](https://peps.python.org/pep-0008/) and [PEP 257](https://peps.python.org/pep-0257/), with Ruff formatting at 88 columns and four-space indentation. The formatter's line length is a target; do not contort URLs, documentation or generated schema bytes to fit it. +- Run Ruff's error, import, common-bug and Python modernization rules (`E4/E7/E9`, `F`, `I`, `B`, `UP`). Import groups follow standard library, third-party and project-local order. Explicit re-exports use `as` or `__all__`; avoid wildcard imports. +- Keep historical public names, import paths and parameter names, including misspellings. New APIs use `snake_case` functions, `CapWords` classes and descriptive argument names. Add useful type annotations to new interfaces; do not require a heavyweight backend to resolve annotations for core tools. +- Document public behavior, units/shape conventions, return values and meaningful errors. Prefer a short docstring over comments that merely restate a line. Do not silently change behavior to satisfy a style rule. +- Prefer explicit validation and scoped exceptions. Preserve exception causes with `raise ... from error`. Do not use bare `except`, swallow failures, or rely on `assert` for public input validation. +- Keep backend imports optional and expose a helpful missing-extra error. Importing `imagedt` must not change eager execution, GPU settings, plotting settings or launch workers. +- Close sessions, figures, files and pools using context managers or explicit `close()`. File cleanup/rename tests must operate only in temporary fixtures. Avoid overwriting an existing destination without a documented contract. +- Generated `caffe_pb2.py` is excluded from formatting/lint because it embeds schema bytes. TensorFlow test import ordering has one documented `E402` exception so the optional backend can skip before dependent imports. Add no broad suppressions merely to make CI pass. + +## Before submitting + +Run the commands in [TESTING.md](TESTING.md). Formatting and lint checks are enforced by CI; optional TensorFlow tests run in a separate integration job. A pass with skipped backends must be reported with its skip count rather than as full backend validation. Explain any changed metric values or checkpoint naming in [docs/migration.md](docs/migration.md). Update README/CHANGELOG when installation or user-visible behavior changes. + +On systems with GNU/BSD Make, `make PYTHON=python lint`, `make PYTHON=python test` and `make PYTHON=python build` wrap the same commands. Native Python commands remain the portable interface, including on Windows. + +## Compatibility and release policy + +The legacy API fixture is a baseline, not an authorization to remove methods. Tests must check behavior as well as name/argument preservation. Numerical bug fixes should state the old and new semantics. Keep optional capability and unsupported-operation boundaries explicit. + +Support is Python 3.10+. Core CI covers 3.10, 3.12 and 3.14 across Linux/macOS/Windows; TensorFlow integration runs on 3.12. Intermediate Python versions are within the support range but are not separate matrix lanes. If a new dependency requires a higher Python version, discuss that compatibility change before merging. + +Do not upload PyPI as part of an ordinary PR. The maintainer reviews the candidate, runs release validation, then explicitly publishes. Existing PyPI versions cannot be overwritten. `bash upload_pip.sh` builds/checks; `--upload` is the explicit publication step. Never include API tokens in a PR or committed config. + +## Community conduct + +Be respectful, specific and evidence-based in issues and reviews. Critique code and behavior rather than people. Harassment, discriminatory remarks and disclosure of personal information are not acceptable. Maintainers may edit or remove abusive content and limit participation. Report sensitive security issues through [SECURITY.md](SECURITY.md), not a public issue with exploit details. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..311137b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include LICENSE README.md CHANGELOG.md CONTRIBUTING.md TESTING.md SECURITY.md Makefile .editorconfig .gitattributes pyproject.toml requirements.txt upload_pip.sh +recursive-include tests *.py *.json +recursive-include test *.py +recursive-include scripts *.py +recursive-include docs *.md *.json +include .github/workflows/test.yml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0c404e4 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +PYTHON ?= python3 + +.PHONY: format lint test coverage build check + +format: + $(PYTHON) -m ruff format imagedt tests scripts test setup.py + $(PYTHON) -m ruff check imagedt tests scripts test --fix + +lint: + $(PYTHON) -m ruff check imagedt tests scripts test + $(PYTHON) -m ruff format --check imagedt tests scripts test setup.py + +test: + $(PYTHON) -m pytest -q + +coverage: + $(PYTHON) -m pytest --cov=imagedt --cov-report=term-missing --cov-fail-under=88 + +build: + $(PYTHON) -m build + $(PYTHON) -m twine check dist/* + $(PYTHON) scripts/check_release.py + +check: lint test diff --git a/README.md b/README.md index dc21714..6ef3e90 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,89 @@ # ImageDT - a python lib for neural networks, file and image processing etc. - -Installation ------------- -1. pip install - pip install imagedt
-2. source install - - #1 pip install imagedt - <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - #2 git clone https://github.com/hanranCode/ImageDT.git - cd ImageDT - python setup.py install - <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - #3 vi ~/.bashrc - export PYTHONPATH="/path_to_ImageDT:$PYTHONPATH" - import imagedt in your project - - -Dependencies ------------- -1. tensorflow
-2. coremltools
-*Coremltools* has the following dependencies: - -- numpy (1.10.0+) -- protobuf (3.1.0+) - -In addition, it has the following soft dependencies that are only needed when -you are converting models of these formats: - -- Keras (1.2.2, 2.0.4+) with corresponding Tensorflow version -- Xgboost (0.7+) -- scikit-learn (0.17+) -- libSVM
-
- - -TODO ------------- -- tools
- - [x] Converte VOC data to tfrecords
- - [ ] Data interface
-- networks
- - - tensorflow - + [x] lenet test - + [ ] ssd_mobilenet
- - - caffe - - [ ] ?
- - - pyroch - - [ ] ??
- \ No newline at end of file + +Python 3 utilities for image processing, files, annotations, metrics, synthetic text, TFRecords and model conversion. The original module paths, function names and public argument names are retained, including historical spellings such as `noise_padd`, `fillter_file_type`, `wirte_xml_file` and `gernerate_infos_dict`. + +## Install + +Requires Python 3.10 or newer. The candidate release is **0.6.0rc1**. + +```bash +python -m pip install imagedt +# Optional capabilities: +python -m pip install 'imagedt[plot,excel,pdf]' +python -m pip install 'imagedt[tensorflow]' # TensorFlow 2 + LiteRT +python -m pip install 'imagedt[lite]' # LiteRT inference without TensorFlow +python -m pip install 'imagedt[caffe]' # Caffe protobuf/graph tooling +``` + +For this checkout, use `python -m pip install -e '.[dev,plot,excel,pdf,tensorflow]'`. The candidate is only available on PyPI after publication. OpenCV's headless package supports processing without requiring a desktop display; `cv_show` needs a GUI-enabled OpenCV installation. Install exactly one OpenCV distribution in an environment. + +Importing `imagedt` does not import TensorFlow, matplotlib, OpenCV or spreadsheet backends. Capabilities load when their exported names are accessed. An unavailable optional backend reports the extra to install. + +## Familiar interfaces + +```python +import imagedt + +paths = imagedt.loop('./images', ['.jpg', '.png']) +imagedt.write_csv([['name', 'count'], ['图片', 3]], 'counts.csv') +boxes = imagedt.Anno_OP.parse_lab_xml('annotations/example.xml') +``` + +`FilesOp`, `Anno_OP`, `RecordWriter` and `plot_tools` remain singleton objects at their historical import paths. Traversal is deterministic. Text files use UTF-8; CSV readers also accept a UTF-8 BOM. + +```python +from imagedt.tensorflow.tools.tfmodel_wrapper import TFmodel_Wrapper + +with TFmodel_Wrapper('model.pb', input_nodename='data', + output_nodename='softmax,features') as model: + classifications = model.predict(images) + features = model.extract(images) +``` + +Graph wrappers own isolated sessions and support `close()` and context managers. Importing a backend does not disable eager execution or change GPU environment variables. TF1 graph APIs use `tf.compat.v1`; queue-based `Data_Provider` should be created within `tf.Graph().as_default()` and run with a Session and queue-runner lifecycle. + +## Behavioral corrections and migration limits + +See [the migration guide](docs/migration.md) for precise changes. Important corrections include class-aware detection matching, filename-aligned classification evaluation, no train/validation leakage, bounded dataset batching, binary XML writing, complete image cleanup, and collision-aware paired renaming. + +Maintained conversion support covers the bundled Caffe layer subset to frozen GraphDef, frozen GraphDef to float TFLite, and float32 single-image NHWC LiteRT classification. Arbitrary Caffe variants or model checkpoints are not guaranteed. LeNet uses Keras layers while retaining endpoint keys; **old TF-Slim variable/checkpoint names are not preserved**. + +These old capabilities retain callable entrances and raise explicit errors: + +- Caffe-to-CoreML: removed from modern coremltools; first migrate the model to a supported format. +- Handwriting generation: the external TF1 pretrained model was never bundled. Computer text generation remains supported. +- Legacy TOCO `GRAPHVIZ_DOT` and uint8 quantization without representative calibration data. + +PDF rendering uses portable PDFium wheels rather than ImageMagick/Ghostscript. Single-page output keeps the requested filename; multiple pages use `-0`, `-1`, etc. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for Python conventions, branches and contribution workflow; [TESTING.md](TESTING.md) for checks and test lanes; [SECURITY.md](SECURITY.md) for private vulnerability reporting. + +## Develop and verify + +```bash +python -m pip install -e '.[dev,plot,excel,pdf,tensorflow]' +python -m ruff check imagedt tests scripts test +python -m ruff format --check imagedt tests scripts test setup.py +python -m pytest --cov=imagedt --cov-report=term-missing +python -m build +python -m twine check dist/* +python scripts/benchmark_utils.py +``` + +Tests generate their own images, XML, fonts and tiny models in temporary directories. No personal `/data` paths, production datasets or GPU are needed. TensorFlow tests skip explicitly when the optional backend is absent. `tests/fixtures/legacy_api.json` captures public names and argument names from the original revision and guards compatibility. It does not claim numerical equivalence for intentionally corrected bugs or unchanged trained-checkpoint compatibility. + +CI defines Python 3.10/3.12/3.14 core jobs on Linux, macOS and Windows, plus a Python 3.12 TensorFlow/LiteRT integration job. Remote CI execution is separate from local verification. + +## Release + +`bash upload_pip.sh` builds into a fresh temporary directory and checks metadata. It does not upload. Publish only after reviewing the artifacts and configuring PyPI credentials: + +```bash +PYTHON=python3 bash upload_pip.sh --upload +``` + +See [release validation](docs/release-readiness.md) and [changes](CHANGELOG.md). License: Apache-2.0. + +The [historical technical review](docs/technical-reviews/2026-10-12-78c9e67/technical-review.md) records the original revision baseline, architecture and migration decisions. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5d0e8b3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security policy + +The maintained branch is `main`. Security fixes target the current Python 3 release line; old Python 2 environments and removed third-party backends are not maintained security targets. Candidate releases should be evaluated against the documented compatibility limits before production use. + +Please do not publish credentials, private datasets or working exploit details in a public issue. Use GitHub's **Report a vulnerability** option in this repository's Security tab when available. If private reporting is unavailable, contact the maintainer through the author contact in the package metadata and ask for a private reporting channel before sending sensitive details. Do not send model files or personal data until the recipient has agreed to receive them. + +A useful report includes affected version, module/function, required conditions, expected impact, and a minimal reproduction using synthetic data. No response-time or bounty program is promised. + +Model conversion and weight loading process caller-supplied files. Only load trusted model/weight artifacts; the legacy NumPy weight format uses pickle-compatible loading. ImageDT is an offline utility library, not a sandbox for hostile models. File deletion and rename tools should run with permissions limited to the intended dataset. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..7e8be56 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,30 @@ +# Testing and development checks + +Use the isolated environment described in [CONTRIBUTING.md](CONTRIBUTING.md). Tests use pytest and temporary self-contained fixtures; they do not need a GPU or personal `/data` directories. + +```bash +python -m ruff check imagedt tests scripts test +python -m ruff format --check imagedt tests scripts test setup.py +python -m pytest -q +python -m pytest --cov=imagedt --cov-report=term-missing --cov-fail-under=88 +python -m build +python -m twine check dist/* +python scripts/check_release.py +``` + +The coverage gate applies to the full optional-backend environment. Installing only core/plot/excel/pdf/caffe extras intentionally skips TensorFlow-specific tests; do not apply a whole-package coverage gate to that smaller environment. Skips remain visible (`-ra`). `caffe_pb2.py` and the duplicate standalone LeNet test are omitted from coverage, while protobuf roundtrips and LeNet behavior are tested in the main suite. + +- `tests/test_core.py`: file/CSV/JSON/XML, directories, image processing, import isolation and timing. +- `tests/test_metrics_data_text.py`: F1/ROC semantics, batching, synthetic text and plotting. +- `tests/test_tensorflow.py`: actual graph/LiteRT conversion and inference, TFRecord write/read, losses and tensor transforms. +- `tests/test_caffe_backends.py`: protobuf/graph/transform behavior and Caffe conversion. +- `tests/test_contracts.py`: original public names/argument names and module import paths. +- `tests/test_remaining_utilities.py`: negative paths, file rename failure injection, PDF rendering and explicit unsupported behavior. + +Use `tmp_path` for files, meaningful numerical/shape assertions for transformations, and mocked network access where the behavior under test is networking error handling. Test both the ordinary result and the relevant failure/boundary. Keep tests deterministic where the API promises deterministic behavior; for stochastic images, check contract properties rather than arbitrary exact pixels. Avoid tests that duplicate implementation details without observing behavior. + +For optional integration fixtures, use `pytest.importorskip` and a declared marker. The intentionally delayed imports in `test_tensorflow.py` are necessary for backend-less environments, not a general exemption from import ordering. + +`python scripts/benchmark_utils.py` checks ROC numerical equivalence and prints timing; timing is informational and is not a flaky CI speed threshold. `python -m imagedt.tensorflow.network.test_lenet` runs the older standalone test entrance. The files under `test/` remain launchers for corresponding maintained tests. + +Before publication, run tests against an installed wheel outside the checkout as well as source tests. The source distribution must contain `tests/fixtures/legacy_api.json` and migration documentation; `scripts/check_release.py` verifies those entries, metadata and wheel compilation. Build in a clean output directory if old versions are present: the checker deliberately rejects ambiguous artifact sets. diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 0000000..81dc489 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,41 @@ +# Python 3 migration: 0.4.1 → 0.6.0rc1 + +## Compatibility policy + +Original public function/class names, module paths and argument names are checked against `tests/fixtures/legacy_api.json`, captured from the original Git revision. Optional implementations may be re-exported from shared modules. Defaults that used mutable lists now use tuples or `None`. The default LeNet prediction callback is resolved on invocation, and produces softmax predictions as before. Accidental exports of third-party module objects (`np`, `os`, `tf`, etc.) are not public compatibility promises. `from imagedt import *` loads all documented exports, including optional plotting; use specific imports if extras are not installed. + +Old test filenames in `test/` now launch maintained tests. Dataset-specific demonstration code was replaced with temporary-fixture regression tests. Caffe/Torch trainer packages were empty placeholders; they remain importable placeholders. + +## Corrected behavior + +| Area | New behavior and reason | +|---|---| +| Directory traversal | Uses `scandir`, sorted names, safe extension extraction, copied filters and no recursion into directory symlinks. Results are reproducible and cyclic links terminate. | +| Image cleanup | Thread workers avoid macOS process-pickling failures. All workers finish before `clear_dir` returns; every file is checked, including leftovers beyond equal-size chunks. | +| Padding | Integer sizes, positive dimension validation, bounded resize dimensions and aspect-preserving rectangular fit. Extremely thin images remain valid. | +| Resize/hash/draw | Requested resize bound is honored; `imwrite` argument order is corrected; binary streaming hashes work on image bytes; polygons become standard OpenCV `(N,4,1,2)` contours. | +| File pairs | Multi-dot stems are preserved, annotation membership checks use a set, pair renames preflight collisions and roll back completed renames on a subsequent failure. MD5 image/XML renames share the same collision preflight and rollback. No filesystem transaction is promised against concurrent writers or process termination. | +| Files | UTF-8 text, BOM-aware CSV with correct newline handling, optional XLS dependency and byte decoding. | +| XML | Binary serialization, disabled external entity resolution, safe defaults, stable empty polygon tuple, consistent list coordinates and list-based mutation. | +| Detection F1 | Uses actual/explicit labels, confidence filtering, class-aware greedy matching and VOC inclusive boxes with strict `IoU > threshold`. Duplicate/unmatched detections count as FP, unmatched non-difficult GT as FN, difficult GT are ignored, empty scores are zero. Class 0 remains excluded. Numeric results may differ because old matching was incorrect. | +| Classification F1 | Aligns input files by basename, rejects duplicate/missing image IDs, keeps the header out of numeric sorting, populates object IDs even with zero true positives, and handles empty totals. | +| ROC | Preserves the 1001 threshold grid and output filenames; sorted probability arrays with `searchsorted` replace the nested Python scan. Rejects invalid probabilities and closes generated figures. This reports confidence-versus-correctness, following the original contract; it is not a general multiclass ROC API. | +| Dataset batches | Preserves shuffle-with-paired-labels, fixed batches discard incomplete tails, non-fixed batches include tails. Empty/mismatched data and impossible fixed batch sizes raise `ValueError` instead of recursing indefinitely. | +| TFRecord | Shared serialization implementation; output directories are created; emitted PNG bytes advertise PNG; no empty extra shard; validation samples are excluded from training; empty detection annotations serialize correctly, corrupt-image reports are split-specific, and converter-generated classification labels use 0..N-1 for one-hot compatibility. Classification regeneration stages new shards before publication and removes obsolete `train_######.tfrecord` / `validation_######.tfrecord` files (exactly six digits), preserving unrelated filenames. Publication across multiple files is not atomic. Existing one-based classification indexes must be remapped before Data_Provider use. | +| Model wrappers | Isolated graphs and explicit session closure; top-class selection uses argmax rather than a complete sort; tied maximum scores select the first class. | +| Caffe graph | Descriptor-pool protobuf, iterative linear-time DAG ordering, updated node lookup after renaming, correct exclude-phase filtering, integer grouped-convolution channels, current split/concat signatures and stable FC/bias dimensions. Serialized BatchNorm statistics are normalized correctly and protobuf Scale gamma is preserved in supported BatchNorm/Scale fusion. Terminal output layers retain a stable graph alias, and unsupported padding/layers fail explicitly. SAME is accepted only when its effective padding equals Caffe symmetric padding; equal output shape alone is insufficient. Generated Python network names are validated and string arguments quoted. | +| LeNet/losses | LeNet uses maintained Keras and retains function/endpoint names. TF-Slim checkpoint variable names change. ArcFace normalizes zero vectors safely and clamps square-root rounding errors. | +| Synthetic text | Pillow metrics/resampling updated; font size, image size and orientation arguments are honored. Reusable thread pools have `close()` and context managers; batches preserve ordering. Quasicrystal backgrounds are vectorized; distortion arrays use uint8. Outputs will not reproduce the old random font-size/half-size digit layout. | +| PDF | PDFium replaces native ImageMagick/Ghostscript requirements. Multiple pages have deterministic indexed filenames. | + +## Explicit unsupported operations + +`caffe_coreml.convert`, `HandwrittenTextGenerator.generate`, `FakeTextDataGenerator.generate(is_handwritten=True)`, old TOCO DOT output, and old uncalibrated uint8 conversion raise `NotImplementedError`. The user selected preservation of old entrances with a modern backend and explicit failure when compatibility cannot be maintained. No emulation of deleted upstream facilities is claimed. + +Model integration tests exercise small synthetic models and the supported layer subset. They do not establish equivalence for every historical Caffe model, custom layer or trained TF-Slim checkpoint. For those workloads, add a representative fixture and compare outputs before switching environments. + +## Backend references + +- [TensorFlow compatibility converter](https://www.tensorflow.org/api_docs/python/tf/compat/v1/lite/TFLiteConverter) +- [Google LiteRT migration](https://developers.google.com/edge/litert/migration) +- [Python packaging guide](https://packaging.python.org/en/latest/tutorials/packaging-projects/) diff --git a/docs/release-readiness.md b/docs/release-readiness.md new file mode 100644 index 0000000..963d0ed --- /dev/null +++ b/docs/release-readiness.md @@ -0,0 +1,52 @@ +# 0.6.0rc1 候选版发布就绪报告 + +本次重构已达到 PyPI **候选版**发布准备状态；尚未上传 PyPI。代码与验证结果将提交候选 PR 审阅。正式稳定版仍应由实际使用者审阅行为变化与模型兼容性范围。 + +## 已执行验证 + +| 环境/检查 | 结果 | +|---|---| +| macOS arm64 / Python 3.12.9，完整后端 | 225 tests passed，无 skip | +| macOS arm64 / Python 3.10.19，核心 + plot/excel/pdf/caffe | 181 passed，26 明确 skip（未安装 TensorFlow） | +| macOS arm64 / Python 3.14.7，核心 + plot/excel/pdf/caffe | 181 passed，26 明确 skip（未安装 TensorFlow） | +| 覆盖 | 行覆盖约 93.4%；分支约 83.4%；合计约 91.3% | +| Ruff / Python 编译 / shell 语法 | 通过 | +| 原版本 API 对照 | 原公共函数、类、方法的名称与参数名检查通过；shared implementations 的 re-export 也检查 | +| 真实集成 | Caffe→冻结图→TFLite/LiteRT；隔离 graph 推理；TFRecord 切分、读取、检测标注；PDF 单页和多页 | +| wheel / sdist 构建 | 成功，Python 3 wheel (`py3-none-any`) | +| twine metadata check | wheel 与 sdist 均 PASSED | +| 全新 wheel 安装 + 源码包测试 | 离开仓库目录运行，181 passed、26 明确 skip；确认从独立环境的 site-packages 导入 | +| 分发包完整性 | 源码包包括接口 fixture、迁移说明与测试；wheel 源码可编译 | +| 架构评估报告 schema | 6 findings，通过技能 validator | + +生成代码 caffe_pb2.py 与重复的独立 test_lenet.py 未纳入覆盖率分母；前者有 protobuf roundtrip 测试，后者另外直接执行。覆盖率不是所有历史模型或每个错误分支已经验证的保证。 + +完整后端的 225 项测试已在最后的 BatchNorm/Scale 融合与 TFRecord 重生成修复后重跑。Python 3.10/3.14 核心环境及独立 wheel 安装记录来自这些后端修复之前，且明确跳过 TensorFlow 集成；不作为最后后端修复的独立环境验证证据。 + +CI 已配置 Linux/macOS/Windows 的 Python 3.10/3.12/3.14 核心矩阵与 Linux Python 3.12 后端集成。远端 CI 尚未执行，因此不声称已验证所有操作系统。 + +## 本机依赖样本 + +NumPy 2.5.4、Pillow 12.3.0、OpenCV headless 5.0.0.93、lxml 6.1.3、TensorFlow 2.21.0、protobuf 7.36.2、LiteRT 2.3.0、PDFium 5.14.0、pytest 9.1.1。这是已测试版本样本，不是对所有下界版本组合逐一验证的声明。 + +## 性能验证 + +`python scripts/benchmark_utils.py` 使用固定随机数据：5000 样本 × 1001 阈值。新旧 ROC 扫描数值逐项一致。本机一次观测：旧扫描约 0.296 秒，新实现约 0.00349 秒（约 84.6 倍）。这里只比较计算阶段，排除绘图/I/O；结果受机器负载影响，不作为跨平台性能承诺。 + +Caffe DAG 排序使用迭代算法，2000 节点链验证不触发递归限制。目录扫描/文件对匹配和背景生成采用 `scandir`、集合和向量运算；没有对这些部分声称未测量的速度提升比例。 + +## PyPI 版本核对 + +2026-10-12 只读核对 PyPI：现有最新版本是 0.5.0；本次版本改为 0.6.0rc1，查询时该版本尚未发布。发布账号权限未核实，版本可用不等于账号已获授权。 + +## 发布范围与限制 + +发布版本为 `0.6.0rc1`，支持 Python 3.10+。已按用户决策保留旧接口并迁移现代后端。Caffe-to-CoreML、缺失资产的 TF1 手写模型、旧 DOT/未校准 uint8 转换明确抛出错误。LeNet 保留函数和 endpoint，但旧 TF-Slim checkpoint 变量名改变。任意真实历史模型或自定义 Caffe 层需要自己的对照 fixture；[迁移说明](migration.md)列明具体行为变化。 + +文件修改/删除工具保留原有用途。配对重命名预检并在正常可处理失败时回滚，不承诺面对并发写入或进程崩溃时具有文件系统事务。 + +## 可审阅产物和发布操作 + +当前产物在 `dist/imagedt-0.6.0rc1-py3-none-any.whl` 与 `dist/imagedt-0.6.0rc1.tar.gz`。`python scripts/check_release.py` 输出完整性检查与 SHA256。`bash upload_pip.sh` 仅构建检查；显式 `--upload` 才上传。本次没有查看 PyPI 凭据或确认账号所有权。 + +发布者审阅后，可用配置好的 PyPI 账号执行 `PYTHON=python3 bash upload_pip.sh --upload`。上传不可覆盖同一版本，源码与构建检查应和本次批准的代码保持一致。 diff --git a/docs/technical-reviews/2026-10-12-78c9e67/architecture-model.md b/docs/technical-reviews/2026-10-12-78c9e67/architecture-model.md new file mode 100644 index 0000000..e7a4171 --- /dev/null +++ b/docs/technical-reviews/2026-10-12-78c9e67/architecture-model.md @@ -0,0 +1,28 @@ +# 项目架构与边界 + +基线是 [review-manifest.json](review-manifest.json) 中的原始 revision；改造结果位于当前工作区。 + +| 能力 | 模块 | 输入/输出 | 外部依赖 | +|---|---|---|---| +| 文件与目录 | file, dir, decorator | 路径、文本、CSV、JSON、XML、XLS | lxml、NumPy；可选 xlwt | +| 图像与合成 | image, tools.text_generator | OpenCV/Pillow 数组和图片 | NumPy、Pillow、OpenCV | +| 评估与绘图/PDF | tools.metrics, plot_curve, pdf | 检测/分类文件，CSV、PNG/JPEG | 可选 matplotlib、PDFium | +| 数据与推理 | tensorflow.tools, tensorflow.lite | 数据批次、TFRecord、冻结图、Lite 模型 | TensorFlow 2 compatibility、LiteRT | +| 模型转换 | converters.taffe, caffe_tfmodel, tfmodel_lite | Caffe protobuf/weights → GraphDef → TFLite | protobuf、TensorFlow；图层子集 | +| 旧框架占位 | caffe, torch, trainer | 原有包路径 | 未实现训练器；不推断其有功能 | + +```mermaid +flowchart LR + Public[历史 Python 入口] --> Lazy[按需导出] + Lazy --> Files[文件 / 目录 / XML] + Lazy --> Images[图像 / 文本生成] + Lazy --> Metrics[指标 / 绘图 / PDF] + Lazy --> ML[模型 / TFRecord] + ML --> TF[TensorFlow 2 compat.v1 / Keras] + ML --> Lite[LiteRT] + ML --> Proto[Caffe protobuf / 图变换] +``` + +关键链路：目录扫描→生成文件对→解析 XML→TFRecord；分类结果/标注→按图片对齐→指标 CSV；Caffe定义/权重→图形状/融合/转置→冻结图→Lite 转换→推理。每条链路有临时 fixture 或合成模型测试。 + +原 README 描述的训练网络大量是 TODO，占位代码不是实际产品能力。原测试使用个人目录且没有可运行的自动验证机制，属于文档/验证漂移。没有服务、数据库、租户、网络业务 API 或部署拓扑证据；不能推断生产 SLA。 diff --git a/docs/technical-reviews/2026-10-12-78c9e67/findings.json b/docs/technical-reviews/2026-10-12-78c9e67/findings.json new file mode 100644 index 0000000..56b8305 --- /dev/null +++ b/docs/technical-reviews/2026-10-12-78c9e67/findings.json @@ -0,0 +1,182 @@ +[ + { + "id": "PTR-ENGINEERING-PYTHON2", + "title": "Python 2 运行语义与陈旧依赖阻碍安装", + "domain": "engineering-foundation", + "severity": "P1", + "confidence": "HIGH", + "evidence_level": "DIRECT", + "status": "RESOLVED", + "evidence": [ + { + "locator": "78c9e671526422f28bd564cad9879ef95f12b454:imagedt/file/file_write.py:write_txt; requirements.txt", + "claim": "unicode 与老版本 NumPy/protobuf，安装元数据没有依赖" + }, + { + "locator": "tests/; docs/migration.md", + "claim": "Python 3.10/3.12/3.14 安装与临时数据回归" + } + ], + "observation": "unicode 与老版本 NumPy/protobuf，安装元数据没有依赖", + "failure_scenario": "Python 3 下调用文本写入失败，图片填充使用浮点尺寸失败", + "impact": [ + "correctness", + "maintainability" + ], + "root_cause": "多年未维护，缺少 Python 版本及依赖契约", + "recommendation": "迁移 Python 3、统一现代元数据并设置版本测试矩阵", + "tradeoffs": "停止 Python 2；最低支持 Python 3.10", + "effort": "M", + "verification": "Python 3.10/3.12/3.14 安装与临时数据回归" + }, + { + "id": "PTR-ARCH-EAGER-OPTIONALS", + "title": "通用入口耦合重型可选依赖", + "domain": "system-architecture", + "severity": "P1", + "confidence": "HIGH", + "evidence_level": "DIRECT", + "status": "RESOLVED", + "evidence": [ + { + "locator": "78c9e671526422f28bd564cad9879ef95f12b454:imagedt/__init__.py; imagedt/tools/__init__.py; imagedt/converters/taffe/__init__.py", + "claim": "多层星号导入加载图像、绘图、Excel 和 TensorFlow" + }, + { + "locator": "tests/; docs/migration.md", + "claim": "独立 subprocess 检查顶层不加载框架；无 TensorFlow 的核心环境运行" + } + ], + "observation": "多层星号导入加载图像、绘图、Excel 和 TensorFlow", + "failure_scenario": "仅调用文件工具也会受未安装框架影响", + "impact": [ + "correctness", + "maintainability" + ], + "root_cause": "模块边界通过 eager star import 连接，没有可选能力分层", + "recommendation": "保留历史名称，按需导入并拆分 extras", + "tradeoffs": "第一次调用时出现可选依赖错误，而非导入包时", + "effort": "M", + "verification": "独立 subprocess 检查顶层不加载框架；无 TensorFlow 的核心环境运行" + }, + { + "id": "PTR-DATA-METRIC-CORRECTNESS", + "title": "指标与数据切分存在正确性错误", + "domain": "product-functional-architecture", + "severity": "P1", + "confidence": "HIGH", + "evidence_level": "DIRECT", + "status": "RESOLVED", + "evidence": [ + { + "locator": "78c9e671526422f28bd564cad9879ef95f12b454:imagedt/tools/metrics/detect_eval.py:voc_eval; imagedt/tools/metrics/f1_score.py:_f1_score; imagedt/tensorflow/tools/Records.py:write_cls_records", + "claim": "固定类别、未匹配误计、按行配对、表头混排、训练包含验证样本" + }, + { + "locator": "tests/; docs/migration.md", + "claim": "完美/重复/错误类别/空框评估；训练验证标签集合不重叠" + } + ], + "observation": "固定类别、未匹配误计、按行配对、表头混排、训练包含验证样本", + "failure_scenario": "模型评估产生误导，甚至验证集泄漏", + "impact": [ + "correctness", + "maintainability" + ], + "root_cause": "缺少边界契约和独立的正确性 fixtures", + "recommendation": "明确评估契约并修复匹配/对齐/切分", + "tradeoffs": "指标数值会因修复而改变", + "effort": "M", + "verification": "完美/重复/错误类别/空框评估；训练验证标签集合不重叠" + }, + { + "id": "PTR-RELIABILITY-RESOURCE-LIFETIME", + "title": "清理及模型资源生命周期不完整", + "domain": "production-architecture", + "severity": "P1", + "confidence": "HIGH", + "evidence_level": "DIRECT", + "status": "RESOLVED", + "evidence": [ + { + "locator": "78c9e671526422f28bd564cad9879ef95f12b454:imagedt/image/process.py:clear_dir; imagedt/tensorflow/tools/tfmodel_wrapper.py:_load_model; imagedt/tools/text_generator/text_generator.py", + "claim": "局部函数作为进程目标、分片遗漏、未 join、默认图污染、池没有关闭契约" + }, + { + "locator": "tests/; docs/migration.md", + "claim": "多余尾文件、两个独立模型、重命名故障注入和复用生成器" + } + ], + "observation": "局部函数作为进程目标、分片遗漏、未 join、默认图污染、池没有关闭契约", + "failure_scenario": "macOS 清理任务失败/返回后未完成；重复加载模型名称冲突", + "impact": [ + "correctness", + "maintainability" + ], + "root_cause": "并发与资源所有权没有显式生命周期", + "recommendation": "线程池等待、隔离图和 close/context-manager，配对重命名预检/回滚", + "tradeoffs": "同步等待完成；配对回滚不提供跨进程事务", + "effort": "M", + "verification": "多余尾文件、两个独立模型、重命名故障注入和复用生成器" + }, + { + "id": "PTR-INTEGRATION-REMOVED-BACKENDS", + "title": "旧框架私有接口已无法直接维护", + "domain": "system-architecture", + "severity": "P1", + "confidence": "HIGH", + "evidence_level": "DIRECT", + "status": "IMPROVED", + "evidence": [ + { + "locator": "78c9e671526422f28bd564cad9879ef95f12b454:imagedt/converters/tfmodel_lite.py; imagedt/tensorflow/lite/lite_wrapper.py; imagedt/converters/caffe_coreml.py", + "claim": "tensorflow.contrib/TOCO 与 CoreML Caffe 旧后端依赖" + }, + { + "locator": "tests/; docs/migration.md", + "claim": "合成 Caffe→GraphDef→TFLite 链路；错误分支；用户已选择现代后端方案" + } + ], + "observation": "tensorflow.contrib/TOCO 与 CoreML Caffe 旧后端依赖", + "failure_scenario": "Python 3 环境转换/推理导入失败", + "impact": [ + "correctness", + "maintainability" + ], + "root_cause": "直接绑定上游历史私有实现", + "recommendation": "迁移公开兼容 API、LiteRT 和 Keras；不存在的能力保留入口明确失败", + "tradeoffs": "TF-Slim 检查点变量名改变；部分功能不再实现", + "effort": "M", + "verification": "合成 Caffe→GraphDef→TFLite 链路；错误分支；用户已选择现代后端方案" + }, + { + "id": "PTR-PERFORMANCE-NESTED-SCANS", + "title": "ROC 与图遍历存在可消除的扫描成本", + "domain": "production-architecture", + "severity": "P2", + "confidence": "HIGH", + "evidence_level": "DIRECT", + "status": "RESOLVED", + "evidence": [ + { + "locator": "78c9e671526422f28bd564cad9879ef95f12b454:imagedt/tools/metrics/roc_curve.py:roc; imagedt/converters/taffe/graph.py:topologically_sorted", + "claim": "ROC 阈值样本嵌套循环，递归排序头插列表" + }, + { + "locator": "tests/; docs/migration.md", + "claim": "5000 样本/1001 阈值与旧算法逐项一致；2000 节点图无递归" + } + ], + "observation": "ROC 阈值样本嵌套循环，递归排序头插列表", + "failure_scenario": "评估样本增长时 Python 扫描开销增长，深图递归溢出", + "impact": [ + "correctness", + "maintainability" + ], + "root_cause": "旧实现没有利用排序索引和线性图算法", + "recommendation": "searchsorted 阈值查询与迭代 DAG 排序，向量化背景", + "tradeoffs": "需要数组排序；多个合法拓扑序的顺序可能不同", + "effort": "M", + "verification": "5000 样本/1001 阈值与旧算法逐项一致；2000 节点图无递归" + } +] diff --git a/docs/technical-reviews/2026-10-12-78c9e67/gate-review.md b/docs/technical-reviews/2026-10-12-78c9e67/gate-review.md new file mode 100644 index 0000000..d00ab69 --- /dev/null +++ b/docs/technical-reviews/2026-10-12-78c9e67/gate-review.md @@ -0,0 +1,14 @@ +# Architecture Gate + +由同一执行者进行独立重读和失败路径审查；没有获得额外子代理授权，未派生代理。 + +| Finding | Outcome | 复核 | +|---|---|---| +| PTR-ENGINEERING-PYTHON2 | ACCEPT | 字符串、字节、整数尺寸的实际回归测试支持结论；不声称 Python 3 语法迁移足以保证旧模型等价。 | +| PTR-ARCH-EAGER-OPTIONALS | ACCEPT | 无 TensorFlow 的独立环境暴露并修复 taffe 的残余 eager 导入。 | +| PTR-DATA-METRIC-CORRECTNESS | ACCEPT | 数学定义、人工小例和数据拆分集合交叉检验；说明指标行为改变。 | +| PTR-RELIABILITY-RESOURCE-LIFETIME | ACCEPT | worker 完成、独立 graph、rollback 故障注入；降级跨进程事务保证。 | +| PTR-INTEGRATION-REMOVED-BACKENDS | REVISE | 真实小模型只验证受支持子集；CoreML/手写模型不能宣称迁移成功，改为明确失败并记录 checkpoint 限制。 | +| PTR-PERFORMANCE-NESTED-SCANS | ACCEPT | 可复现算法比较验证数值，时间仅为本机结果；不推断线上吞吐。 | + +跨平台 CI 和真实历史模型 equivalence 没有执行证据，明确列为证据限制；未引入新的服务、调度或分布式组件。 diff --git a/docs/technical-reviews/2026-10-12-78c9e67/review-manifest.json b/docs/technical-reviews/2026-10-12-78c9e67/review-manifest.json new file mode 100644 index 0000000..a89be67 --- /dev/null +++ b/docs/technical-reviews/2026-10-12-78c9e67/review-manifest.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "review_id": "2026-10-12-78c9e67", + "mode": "initial", + "generated_at": "2026-10-12T11:07:59.472004+08:00", + "repository": "https://github.com/hanranCode/ImageDT", + "revision": "78c9e671526422f28bd564cad9879ef95f12b454", + "previous_review_id": null, + "scope": [ + "entire Python package, legacy tests, dependencies and release tooling" + ], + "exclusions": [], + "evidence_limitations": [ + "No production datasets, historical trained checkpoints or deployment telemetry were supplied.", + "Local verification is macOS arm64; configured Linux/Windows CI has not executed remotely.", + "Original revision is the baseline; implementation outcomes refer to the current uncommitted working tree." + ] +} diff --git a/docs/technical-reviews/2026-10-12-78c9e67/technical-review.md b/docs/technical-reviews/2026-10-12-78c9e67/technical-review.md new file mode 100644 index 0000000..e2d9c4a --- /dev/null +++ b/docs/technical-reviews/2026-10-12-78c9e67/technical-review.md @@ -0,0 +1,96 @@ +# 全项目技术评估与 Python 3 重构 + +## Executive Summary + +ImageDT 是约 1 万行的历史工具包，其中约一半为生成的 Caffe protobuf 代码。核心价值是文件/图像工具、标注、指标、文本生成和轻量模型链路，而不是现成的训练平台。问题集中在旧运行时、入口耦合和缺少正确性测试。已采用保留历史公共名称的增量迁移方案，不引入新架构框架。 + +用户已决策：保留旧入口，迁移现代后端，无法兼容的功能明确报错。结果与限制见 [迁移指南](../../migration.md)。 + +## Decision Summary + +基础代码已迁移 Python 3，隔离可选依赖，补齐自动测试和构建；Caffe protobuf 改为 descriptor pool，保留 schema；采用 TF compatibility、Keras、LiteRT 和 PDFium。不存在的 CoreML Caffe 和未提供资产的手写模型入口抛出明确错误。LeNet checkpoint 名称不能保证兼容。 + +## Prioritized Roadmap + +| 顺序 | 关联 finding | 负责人类型 | 结果与验证 | +|---|---|---|---| +| 1 已实施 | PTR-ENGINEERING-PYTHON2, PTR-ARCH-EAGER-OPTIONALS | 包维护者 | 最低/最新 Python 安装验证、模块/API 契约 | +| 2 已实施 | PTR-DATA-METRIC-CORRECTNESS, PTR-RELIABILITY-RESOURCE-LIFETIME | 包维护者 | 边界回归、文件故障注入与合成模型链路 | +| 3 已实施 | PTR-PERFORMANCE-NESTED-SCANS | 包维护者 | 可复现 ROC 数值/性能比较与深图排序测试 | +| 4 候选发布 | PTR-INTEGRATION-REMOVED-BACKENDS | 发布者 | 审阅迁移限制，wheel/sdist 构建、metadata 检查、全新安装 | +| 5 实际采用时 | PTR-INTEGRATION-REMOVED-BACKENDS | 模型使用者 | 增加真实历史模型输入/输出 fixture；原环境作为对照，可回退旧包版本 | + +## Scope, Context, and Evidence Limitations + +审阅整个仓库：运行组件、打包、依赖、测试和发布脚本。旧 Git revision 与当前工作区分开记录。没有生产数据、用户使用频率、实际旧模型、业务 SLA 或性能遥测。跨平台配置不是跨平台成功记录。PyPI 所有权和账号凭据未查询，未上传。 + +## Project Profile and Capability Map + +见 [架构模型](architecture-model.md)。它是进程内 Python 库，无已实现的服务或数据存储。caffe/torch 的 trainer 为空，不把 TODO 计为已实现能力。 + +## Current Architecture and Architecture Drift + +原星号导入使依赖边界消失；当前入口是按需导出，内部执行仍为普通函数和轻量对象。文件和图像功能共享目录扫描；两份 TFRecord 实现已合并。没有微服务拆分的现实依据。 + +## Architecture Scorecard + +| 维度 | 基线→当前 | 置信度与证据 | +|---|---|---| +| 工程基础 | 1→4 | HIGH：版本矩阵、可重复 fixtures、构建元数据 | +| 模块边界 | 2→4 | HIGH：懒加载、extras、独立核心安装 | +| 正确性与可靠性 | 1→4 | HIGH：错误结果修复与回归；真实历史模型仍有盲区 | +| 性能 | 2→4（被测工具链） | HIGH：ROC 等价比较；生产吞吐 INSUFFICIENT EVIDENCE | +| 安全性 | 2→3（输入/文件边界） | MEDIUM：XML entity 关闭、生成源码名称约束；不作为全面依赖漏洞审计结论 | +| 部署/运维 SLA、成本 | NOT REVIEWED / N/A | 仅库；缺少生产记录 | + +## Root Cause Map + +长期缺少运行时与接口契约 → Python 2 和过时 backend 绑定 → 导入/运行失败；缺少自包含正确性 fixtures → 指标错误、数据泄漏和生命周期缺陷无法自动发现。以接口基线、依赖隔离和临时数据测试作为共同补救，避免只做格式化。 + +## Technical Debt Map + +代码债：重复序列化、循环扫描、共享默认值；结构债：跨层 eager 导入；过程债：个人目录测试、缺少发行检查；集成债：删除的上游 API 和未提供模型资产。前三项已处理；最后一项通过明确支持范围和失败契约控制，不能宣称恢复不存在的能力。 + +## Evolution Constraints + +任意 Caffe 自定义层、旧 TF-Slim checkpoint 和量化校准都需要具体工作负载 fixture；没有资产就不能保证恢复。保持模块接口比引入新插件系统更符合当前规模。 + +## Product / Platform Opportunities + +现有 XML/图像/TFRecord 原语可形成更一致的离线数据处理工具，但当前没有需求证据，未扩展为训练平台或代理服务。 + +## Detailed Findings + +完整结构化证据与建议见 [findings.json](findings.json)。 + +### PTR-ENGINEERING-PYTHON2 — Python 2 运行语义与陈旧依赖阻碍安装 + +P1 / HIGH / RESOLVED。unicode 与老版本 NumPy/protobuf，安装元数据没有依赖。根因：多年未维护，缺少 Python 版本及依赖契约。处理：迁移 Python 3、统一现代元数据并设置版本测试矩阵。取舍：停止 Python 2；最低支持 Python 3.10。验证：Python 3.10/3.12/3.14 安装与临时数据回归。 + +### PTR-ARCH-EAGER-OPTIONALS — 通用入口耦合重型可选依赖 + +P1 / HIGH / RESOLVED。多层星号导入加载图像、绘图、Excel 和 TensorFlow。根因：模块边界通过 eager star import 连接，没有可选能力分层。处理：保留历史名称，按需导入并拆分 extras。取舍：第一次调用时出现可选依赖错误，而非导入包时。验证：独立 subprocess 检查顶层不加载框架；无 TensorFlow 的核心环境运行。 + +### PTR-DATA-METRIC-CORRECTNESS — 指标与数据切分存在正确性错误 + +P1 / HIGH / RESOLVED。固定类别、未匹配误计、按行配对、表头混排、训练包含验证样本。根因：缺少边界契约和独立的正确性 fixtures。处理：明确评估契约并修复匹配/对齐/切分。取舍：指标数值会因修复而改变。验证：完美/重复/错误类别/空框评估；训练验证标签集合不重叠。 + +### PTR-RELIABILITY-RESOURCE-LIFETIME — 清理及模型资源生命周期不完整 + +P1 / HIGH / RESOLVED。局部函数作为进程目标、分片遗漏、未 join、默认图污染、池没有关闭契约。根因：并发与资源所有权没有显式生命周期。处理：线程池等待、隔离图和 close/context-manager，配对重命名预检/回滚。取舍：同步等待完成；配对回滚不提供跨进程事务。验证：多余尾文件、两个独立模型、重命名故障注入和复用生成器。 + +### PTR-INTEGRATION-REMOVED-BACKENDS — 旧框架私有接口已无法直接维护 + +P1 / HIGH / IMPROVED。tensorflow.contrib/TOCO 与 CoreML Caffe 旧后端依赖。根因：直接绑定上游历史私有实现。处理：迁移公开兼容 API、LiteRT 和 Keras；不存在的能力保留入口明确失败。取舍：TF-Slim 检查点变量名改变；部分功能不再实现。验证：合成 Caffe→GraphDef→TFLite 链路；错误分支；用户已选择现代后端方案。 + +### PTR-PERFORMANCE-NESTED-SCANS — ROC 与图遍历存在可消除的扫描成本 + +P2 / HIGH / RESOLVED。ROC 阈值样本嵌套循环，递归排序头插列表。根因：旧实现没有利用排序索引和线性图算法。处理：searchsorted 阈值查询与迭代 DAG 排序，向量化背景。取舍：需要数组排序；多个合法拓扑序的顺序可能不同。验证：5000 样本/1001 阈值与旧算法逐项一致；2000 节点图无递归。 + +## Measurements and Access Required + +最终本机验证记录见 [发布就绪报告](../../release-readiness.md)。性能样本、源码版本和完整测试结果应一起解释。若用户要保证旧模型结果，需要提供可分发的模型、样例输入和旧环境输出；若要正式上传需要发布者的 PyPI 授权。未请求或读取账号密钥。 + +## Architecture Gate Summary + +见 [gate-review.md](gate-review.md)。高风险项按具体失败场景复核，删除“所有旧模型已兼容”一类没有证据的结论。当前是候选版发布准备与明确的能力边界，仍需发布者审核实际迁移约束。 diff --git a/imagedt/__init__.py b/imagedt/__init__.py index 5873f74..8ffc490 100644 --- a/imagedt/__init__.py +++ b/imagedt/__init__.py @@ -1,16 +1,73 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from .image import * -from .caffe import * -from .tensorflow import * -from .torch import * -from . import converters -from .file import * -from .decorator import * -from .dir import * -from .tools import * - -__version__ = '0.4.1' +"""Compatibility exports, loaded on first use.""" +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "alpha_to_color": (".image", "alpha_to_color"), + "IMG_EXTENSIONS": (".image", "IMG_EXTENSIONS"), + "clear_dir": (".image", "clear_dir"), + "noise_padd": (".image", "noise_padd"), + "remove_broken_image": (".image", "remove_broken_image"), + "resize_with_scale": (".image", "resize_with_scale"), + "padd_pixel": (".image", "padd_pixel"), + "reduce_imagenet_mean": (".image", "reduce_imagenet_mean"), + "swap_chanel_to_RGB": (".image", "swap_chanel_to_RGB"), + "vgg_preprocesing": (".image", "vgg_preprocesing"), + "inception_preprocesing": (".image", "inception_preprocesing"), + "our_processing": (".image", "our_processing"), + "md5": (".image", "md5"), + "get_box": (".image", "get_box"), + "put_cvtext": (".image", "put_cvtext"), + "draw_cvrect": (".image", "draw_cvrect"), + "cv_show": (".image", "cv_show"), + "resize_dir_images_with_scale": (".image", "resize_dir_images_with_scale"), + "points_to_contour": (".image", "points_to_contour"), + "points_to_contours": (".image", "points_to_contours"), + "put_text": (".image", "put_text"), + "draw_contours": (".image", "draw_contours"), + "polly_points2contours": (".image", "polly_points2contours"), + "rename_imfiles_with_md5": (".image", "rename_imfiles_with_md5"), + "DIR_TYPR": (".file", "DIR_TYPR"), + "FilesOp": (".file", "FilesOp"), + "Anno_OP": (".file", "Anno_OP"), + "write_txt": (".file", "write_txt"), + "write_csv": (".file", "write_csv"), + "read_csv": (".file", "read_csv"), + "readlines": (".file", "readlines"), + "write_json": (".file", "write_json"), + "read_json": (".file", "read_json"), + "decode_strs": (".file", "decode_strs"), + "writeExcel": (".file", "writeExcel"), + "LoopFile": (".dir", "LoopFile"), + "loop": (".dir", "loop"), + "get_dir_images": (".dir", "get_dir_images"), + "time_cost": (".decorator", "time_cost"), + "getip": (".tools", "getip"), + "detect_eval": (".tools", "detect_eval"), + "f1_score": (".tools", "f1_score"), + "plot_tools": (".tools", "plot_tools"), + "text_generator": (".tools", "text_generator"), + "pdf": (".tools", "pdf"), + "metrics": (".tools", "metrics"), + "image": (".image", None), + "file": (".file", None), + "dir": (".dir", None), + "decorator": (".decorator", None), + "tools": (".tools", None), + "caffe": (".caffe", None), + "tensorflow": (".tensorflow", None), + "torch": (".torch", None), + "converters": (".converters", None), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) + + +__version__ = "0.6.0rc1" diff --git a/imagedt/_backends.py b/imagedt/_backends.py new file mode 100644 index 0000000..b267797 --- /dev/null +++ b/imagedt/_backends.py @@ -0,0 +1,16 @@ +"""Optional backend loading without changing the caller's global runtime.""" + +from importlib import import_module + + +def require(module, extra): + try: + return import_module(module) + except ModuleNotFoundError as error: + if error.name == module or module.startswith(error.name + "."): + raise ImportError(f"Install imagedt[{extra}] to use {module}") from error + raise + + +def tensorflow_v1(): + return require("tensorflow", "tensorflow").compat.v1 diff --git a/imagedt/_lazy.py b/imagedt/_lazy.py new file mode 100644 index 0000000..fe428b9 --- /dev/null +++ b/imagedt/_lazy.py @@ -0,0 +1,13 @@ +"""Shared lazy export resolver; optional backends load only when requested.""" + +from importlib import import_module + + +def resolve(name, exports, namespace, package): + if name not in exports: + raise AttributeError(f"module {package!r} has no attribute {name!r}") + module_name, attribute = exports[name] + module = import_module(module_name, package) + value = getattr(module, attribute) if attribute else module + namespace[name] = value + return value diff --git a/imagedt/caffe/__init__.py b/imagedt/caffe/__init__.py index c4f40a0..2c4a788 100644 --- a/imagedt/caffe/__init__.py +++ b/imagedt/caffe/__init__.py @@ -1,8 +1,4 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from . import network -from . import optim -from . import tools -from . import trainer \ No newline at end of file +from . import network as network +from . import optim as optim +from . import tools as tools +from . import trainer as trainer diff --git a/imagedt/caffe/network/__init__.py b/imagedt/caffe/network/__init__.py index 760c836..e69de29 100644 --- a/imagedt/caffe/network/__init__.py +++ b/imagedt/caffe/network/__init__.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function diff --git a/imagedt/caffe/optim/__init__.py b/imagedt/caffe/optim/__init__.py index 760c836..e69de29 100644 --- a/imagedt/caffe/optim/__init__.py +++ b/imagedt/caffe/optim/__init__.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function diff --git a/imagedt/caffe/tools/__init__.py b/imagedt/caffe/tools/__init__.py index 760c836..e69de29 100644 --- a/imagedt/caffe/tools/__init__.py +++ b/imagedt/caffe/tools/__init__.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function diff --git a/imagedt/caffe/trainer.py b/imagedt/caffe/trainer.py index 913ef8f..e69de29 100644 --- a/imagedt/caffe/trainer.py +++ b/imagedt/caffe/trainer.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function \ No newline at end of file diff --git a/imagedt/converters/__init__.py b/imagedt/converters/__init__.py index 644d71c..1ddcb2b 100644 --- a/imagedt/converters/__init__.py +++ b/imagedt/converters/__init__.py @@ -1,8 +1,19 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -# from . import caffe_coreml -# from . import caffe_tfmodel -# from . import tfrecords -# from . import tfmodel_lite \ No newline at end of file +"""Compatibility exports, loaded on first use.""" + +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "caffe_coreml": (".caffe_coreml", None), + "caffe_tfmodel": (".caffe_tfmodel", None), + "tfrecords": (".tfrecords", None), + "tfmodel_lite": (".tfmodel_lite", None), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/converters/caffe_coreml.py b/imagedt/converters/caffe_coreml.py index c1ad1d0..1a5f460 100644 --- a/imagedt/converters/caffe_coreml.py +++ b/imagedt/converters/caffe_coreml.py @@ -1,42 +1,18 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Retained Caffe-to-CoreML entry with an explicit unsupported-backend error.""" -import os -import coremltools -# try: -# import coremltools -# except ImportError: -# os.system("pip install coremltools") -# finally: -# print("No module named coremltools.") - - -def convert(model_path, prototxt_path, label_path, red=-123, green=-117, blue=-104, - scale=1.0, bgr=False, output_model_name='sku_cls_model_noise.mlmodel'): - """ - Args: red, grenn, blue should be nagative: like -123, -117, -104 - output_model_name: default, sku_cls_model_noise.mlmodel - label_path: Classes are sorted by incrementing。 - liby_0, 0 - liby_1, 1 - liby_2, 2 - liby_3, 3 - liby_4, 4 - Returns: auto-saved coreml model in source path. - """ - - coreml_model = coremltools.converters.caffe.convert(( - model_path, prototxt_path), - image_input_names='data', - is_bgr=bgr, - image_scale=scale, - red_bias= red*scale, - green_bias=green*scale, - blue_bias=blue*scale, - class_labels=label_path, +def convert( + model_path, + prototxt_path, + label_path, + red=-123, + green=-117, + blue=-104, + scale=1.0, + bgr=False, + output_model_name="sku_cls_model_noise.mlmodel", +): + raise NotImplementedError( + "Modern coremltools removed the Caffe converter. " + "Convert the model to a supported PyTorch or TensorFlow format first." ) - coreml_model.save(os.path.join(os.path.dirname(model_path), output_model_name)) - print("finished converte {0} to {1}".format(os.path.basename(model_path), output_model_name)) - return diff --git a/imagedt/converters/caffe_tfmodel.py b/imagedt/converters/caffe_tfmodel.py index 8622547..027bebb 100644 --- a/imagedt/converters/caffe_tfmodel.py +++ b/imagedt/converters/caffe_tfmodel.py @@ -1,32 +1,34 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import os -import sys + import numpy as np + +from imagedt._backends import tensorflow_v1 + from .taffe import KaffeError, print_stderr from .taffe.tensorflow import TensorFlowTransformer -import shutil -import tensorflow as tf -from tensorflow.python.tools.freeze_graph import freeze_graph +tf = tensorflow_v1() def fatal_error(msg): print_stderr(msg) - exit(-1) + raise KaffeError(msg) def assert_exist_files(*files): for check_file in files: # print("check_file: {0}".format(check_file)) if not os.path.isfile(check_file): - print("{0} not file".format(check_file)) - exit(-1) + print(f"{check_file} not file") + raise FileNotFoundError(check_file) -def convert(prototxt_path, caffemodel_path, phase='test', output_model_name='freeze_graph_model.pb'): +def convert( + prototxt_path, + caffemodel_path, + phase="test", + output_model_name="freeze_graph_model.pb", +): """ 'def_path', 'Model definition (.prototxt) path' 'caffemodel', 'Model data (.caffemodel) path' @@ -35,90 +37,51 @@ def convert(prototxt_path, caffemodel_path, phase='test', output_model_name='fre """ assert_exist_files(prototxt_path, caffemodel_path) - data_output_path = os.path.join(os.path.dirname(prototxt_path), 'output.mat') - code_output_path = os.path.join(os.path.dirname(prototxt_path), 'output.py') - standalone_output_path = os.path.join(os.path.dirname(prototxt_path), output_model_name) - - try: - with tf.Session() as sess: - transformer = TensorFlowTransformer(prototxt_path, caffemodel_path, phase=phase) - print_stderr('Converting data...') - if data_output_path is not None: - data = transformer.transform_data() - print_stderr('Saving data...') - with open(data_output_path, 'wb') as data_out: - np.save(data_out, data) - if code_output_path is not None: - print_stderr('Saving source...') - with open(code_output_path, 'wb') as src_out: - src_out.write(transformer.transform_source()) - - if standalone_output_path: - filename, _ = os.path.splitext(os.path.basename(standalone_output_path)) - temp_folder = os.path.join(os.path.dirname(standalone_output_path), 'tmp') - - if not os.path.exists(temp_folder): - os.makedirs(temp_folder) - - if data_output_path is None: - data = transformer.transform_data() - print_stderr('Saving data...') - data_output_path = os.path.join(temp_folder, filename) + '.npy' - with open(data_output_path, 'wb') as data_out: - np.save(data_out, data) - - if code_output_path is None: - print_stderr('Saving source...') - code_output_path = os.path.join(temp_folder, filename) + '.py' - with open(code_output_path, 'wb') as src_out: - src_out.write(transformer.transform_source()) - - checkpoint_path = os.path.join(temp_folder, filename + '.ckpt') - graph_name = os.path.basename(standalone_output_path) - graph_folder = os.path.dirname(standalone_output_path) - input_node = transformer.graph.nodes[0].name - output_node = transformer.graph.nodes[-1].name - tensor_shape = transformer.graph.get_node(input_node).output_shape - tensor_shape_list = [tensor_shape.batch_size, tensor_shape.height, tensor_shape.width, tensor_shape.channels] - - sys.path.append(os.path.dirname(code_output_path)) - module = os.path.splitext(os.path.basename(code_output_path))[0] - class_name = transformer.graph.name - KaffeNet = getattr(__import__(module), class_name) - - data_placeholder = tf.placeholder(tf.float32, tensor_shape_list, name=input_node) - net = KaffeNet({input_node: data_placeholder}) - - # load weights stored in numpy format - net.load(data_output_path, sess) - - print_stderr('Saving checkpoint...') - - saver = tf.train.Saver() - saver.save(sess, checkpoint_path) - - print_stderr('Saving graph definition as protobuf...') - tf.train.write_graph(sess.graph.as_graph_def(), graph_folder, graph_name, as_text=False) - - input_graph_path = standalone_output_path - input_saver_def_path = "" - input_binary = True - input_checkpoint_path = checkpoint_path - output_node_names = output_node - restore_op_name = 'save/restore_all' - filename_tensor_name = 'save/Const:0' - output_graph_path = standalone_output_path - clear_devices = True - - print_stderr('Saving standalone model...') - freeze_graph(input_graph_path, input_saver_def_path, - input_binary, input_checkpoint_path, - output_node_names, restore_op_name, - filename_tensor_name, output_graph_path, - clear_devices, '') - - shutil.rmtree(temp_folder) - - print_stderr('Done.') - except KaffeError as err: - fatal_error('Error encountered: {}'.format(err)) \ No newline at end of file + transformer = TensorFlowTransformer(prototxt_path, caffemodel_path, phase=phase) + data = transformer.transform_data() + source = transformer.transform_source() + directory = os.path.dirname(prototxt_path) + data_output_path = os.path.join(directory, "output.mat") + with open(data_output_path, "wb") as stream: + np.save(stream, data) + with open(os.path.join(directory, "output.py"), "w", encoding="utf-8") as stream: + stream.write(source) + namespace = {} + exec(compile(source, "", "exec"), namespace) + network_class = namespace[transformer.graph.name] + inputs = transformer.graph.get_input_nodes() + outputs = transformer.graph.get_output_nodes() + if len(inputs) != 1 or len(outputs) != 1: + raise KaffeError( + "Caffe conversion currently requires exactly one input and one output" + ) + input_node = inputs[0].name + output_node = outputs[0].name + shape = transformer.graph.get_node(input_node).output_shape + graph = tf.Graph() + with graph.as_default(), tf.Session(graph=graph) as session: + placeholder = tf.placeholder( + tf.float32, + [shape.batch_size, shape.height, shape.width, shape.channels], + name=input_node, + ) + network = network_class({input_node: placeholder}) + session.run(tf.global_variables_initializer()) + network.load(data_output_path, session) + output_tensor = network.layers[output_node] + frozen = tf.graph_util.convert_variables_to_constants( + session, graph.as_graph_def(), [output_tensor.op.name] + ) + # Variable scopes reserve names even when no op has the bare layer name. + # Add a stable public alias after freezing instead of accepting an auto suffix. + if output_tensor.op.name != output_node: + if any(node.name == output_node for node in frozen.node): + raise KaffeError( + f"Output alias would collide with an existing op: {output_node}" + ) + alias = frozen.node.add(name=output_node, op="Identity") + alias.input.append(output_tensor.name) + alias.attr["T"].type = output_tensor.dtype.as_datatype_enum + output = os.path.join(directory, output_model_name) + with open(output, "wb") as stream: + stream.write(frozen.SerializeToString()) diff --git a/imagedt/converters/taffe/__init__.py b/imagedt/converters/taffe/__init__.py index 96af069..e2848fc 100644 --- a/imagedt/converters/taffe/__init__.py +++ b/imagedt/converters/taffe/__init__.py @@ -1,4 +1,20 @@ -from .graph import GraphBuilder, NodeMapper -from .errors import KaffeError, print_stderr +"""Caffe graph utilities; TensorFlow loads only when its backend is requested.""" -from . import tensorflow +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "GraphBuilder": (".graph", "GraphBuilder"), + "NodeMapper": (".graph", "NodeMapper"), + "KaffeError": (".errors", "KaffeError"), + "print_stderr": (".errors", "print_stderr"), + "tensorflow": (".tensorflow", None), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/converters/taffe/caffe/__init__.py b/imagedt/converters/taffe/caffe/__init__.py index 8d53dee..073844b 100644 --- a/imagedt/converters/taffe/caffe/__init__.py +++ b/imagedt/converters/taffe/caffe/__init__.py @@ -1 +1,2 @@ -from .resolver import get_caffe_resolver, has_pycaffe +from .resolver import get_caffe_resolver as get_caffe_resolver +from .resolver import has_pycaffe as has_pycaffe diff --git a/imagedt/converters/taffe/caffe/caffe_pb2.py b/imagedt/converters/taffe/caffe/caffe_pb2.py index c7583c6..0aad4ff 100644 --- a/imagedt/converters/taffe/caffe/caffe_pb2.py +++ b/imagedt/converters/taffe/caffe/caffe_pb2.py @@ -1,5479 +1,7 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: caffe.proto +# Caffe schema migrated from the original generated descriptor (caffe.proto). +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf.internal import builder as _builder -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='caffe.proto', - package='caffe', - serialized_pb='\n\x0b\x63\x61\x66\x66\x65.proto\x12\x05\x63\x61\x66\x66\x65\"\x1c\n\tBlobShape\x12\x0f\n\x03\x64im\x18\x01 \x03(\x03\x42\x02\x10\x01\"\xcc\x01\n\tBlobProto\x12\x1f\n\x05shape\x18\x07 \x01(\x0b\x32\x10.caffe.BlobShape\x12\x10\n\x04\x64\x61ta\x18\x05 \x03(\x02\x42\x02\x10\x01\x12\x10\n\x04\x64iff\x18\x06 \x03(\x02\x42\x02\x10\x01\x12\x17\n\x0b\x64ouble_data\x18\x08 \x03(\x01\x42\x02\x10\x01\x12\x17\n\x0b\x64ouble_diff\x18\t \x03(\x01\x42\x02\x10\x01\x12\x0e\n\x03num\x18\x01 \x01(\x05:\x01\x30\x12\x13\n\x08\x63hannels\x18\x02 \x01(\x05:\x01\x30\x12\x11\n\x06height\x18\x03 \x01(\x05:\x01\x30\x12\x10\n\x05width\x18\x04 \x01(\x05:\x01\x30\"2\n\x0f\x42lobProtoVector\x12\x1f\n\x05\x62lobs\x18\x01 \x03(\x0b\x32\x10.caffe.BlobProto\"\x81\x01\n\x05\x44\x61tum\x12\x10\n\x08\x63hannels\x18\x01 \x01(\x05\x12\x0e\n\x06height\x18\x02 \x01(\x05\x12\r\n\x05width\x18\x03 \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\r\n\x05label\x18\x05 \x01(\x05\x12\x12\n\nfloat_data\x18\x06 \x03(\x02\x12\x16\n\x07\x65ncoded\x18\x07 \x01(\x08:\x05\x66\x61lse\"\x8a\x02\n\x0f\x46illerParameter\x12\x16\n\x04type\x18\x01 \x01(\t:\x08\x63onstant\x12\x10\n\x05value\x18\x02 \x01(\x02:\x01\x30\x12\x0e\n\x03min\x18\x03 \x01(\x02:\x01\x30\x12\x0e\n\x03max\x18\x04 \x01(\x02:\x01\x31\x12\x0f\n\x04mean\x18\x05 \x01(\x02:\x01\x30\x12\x0e\n\x03std\x18\x06 \x01(\x02:\x01\x31\x12\x12\n\x06sparse\x18\x07 \x01(\x05:\x02-1\x12\x42\n\rvariance_norm\x18\x08 \x01(\x0e\x32#.caffe.FillerParameter.VarianceNorm:\x06\x46\x41N_IN\"4\n\x0cVarianceNorm\x12\n\n\x06\x46\x41N_IN\x10\x00\x12\x0b\n\x07\x46\x41N_OUT\x10\x01\x12\x0b\n\x07\x41VERAGE\x10\x02\"\x8e\x02\n\x0cNetParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x03 \x03(\t\x12%\n\x0binput_shape\x18\x08 \x03(\x0b\x32\x10.caffe.BlobShape\x12\x11\n\tinput_dim\x18\x04 \x03(\x05\x12\x1d\n\x0e\x66orce_backward\x18\x05 \x01(\x08:\x05\x66\x61lse\x12\x1e\n\x05state\x18\x06 \x01(\x0b\x32\x0f.caffe.NetState\x12\x19\n\ndebug_info\x18\x07 \x01(\x08:\x05\x66\x61lse\x12$\n\x05layer\x18\x64 \x03(\x0b\x32\x15.caffe.LayerParameter\x12\'\n\x06layers\x18\x02 \x03(\x0b\x32\x17.caffe.V1LayerParameter\"\x9c\n\n\x0fSolverParameter\x12\x0b\n\x03net\x18\x18 \x01(\t\x12&\n\tnet_param\x18\x19 \x01(\x0b\x32\x13.caffe.NetParameter\x12\x11\n\ttrain_net\x18\x01 \x01(\t\x12\x10\n\x08test_net\x18\x02 \x03(\t\x12,\n\x0ftrain_net_param\x18\x15 \x01(\x0b\x32\x13.caffe.NetParameter\x12+\n\x0etest_net_param\x18\x16 \x03(\x0b\x32\x13.caffe.NetParameter\x12$\n\x0btrain_state\x18\x1a \x01(\x0b\x32\x0f.caffe.NetState\x12#\n\ntest_state\x18\x1b \x03(\x0b\x32\x0f.caffe.NetState\x12\x11\n\ttest_iter\x18\x03 \x03(\x05\x12\x18\n\rtest_interval\x18\x04 \x01(\x05:\x01\x30\x12 \n\x11test_compute_loss\x18\x13 \x01(\x08:\x05\x66\x61lse\x12!\n\x13test_initialization\x18 \x01(\x08:\x04true\x12\x0f\n\x07\x62\x61se_lr\x18\x05 \x01(\x02\x12\x0f\n\x07\x64isplay\x18\x06 \x01(\x05\x12\x17\n\x0c\x61verage_loss\x18! \x01(\x05:\x01\x31\x12\x10\n\x08max_iter\x18\x07 \x01(\x05\x12\x14\n\titer_size\x18$ \x01(\x05:\x01\x31\x12\x11\n\tlr_policy\x18\x08 \x01(\t\x12\r\n\x05gamma\x18\t \x01(\x02\x12\r\n\x05power\x18\n \x01(\x02\x12\x10\n\x08momentum\x18\x0b \x01(\x02\x12\x14\n\x0cweight_decay\x18\x0c \x01(\x02\x12\x1f\n\x13regularization_type\x18\x1d \x01(\t:\x02L2\x12\x10\n\x08stepsize\x18\r \x01(\x05\x12\x11\n\tstepvalue\x18\" \x03(\x05\x12\x1a\n\x0e\x63lip_gradients\x18# \x01(\x02:\x02-1\x12\x13\n\x08snapshot\x18\x0e \x01(\x05:\x01\x30\x12\x17\n\x0fsnapshot_prefix\x18\x0f \x01(\t\x12\x1c\n\rsnapshot_diff\x18\x10 \x01(\x08:\x05\x66\x61lse\x12K\n\x0fsnapshot_format\x18% \x01(\x0e\x32%.caffe.SolverParameter.SnapshotFormat:\x0b\x42INARYPROTO\x12;\n\x0bsolver_mode\x18\x11 \x01(\x0e\x32!.caffe.SolverParameter.SolverMode:\x03GPU\x12\x14\n\tdevice_id\x18\x12 \x01(\x05:\x01\x30\x12\x17\n\x0brandom_seed\x18\x14 \x01(\x03:\x02-1\x12\x11\n\x04type\x18( \x01(\t:\x03SGD\x12\x14\n\x05\x64\x65lta\x18\x1f \x01(\x02:\x05\x31\x65-08\x12\x18\n\tmomentum2\x18\' \x01(\x02:\x05\x30.999\x12\x11\n\trms_decay\x18& \x01(\x02\x12\x19\n\ndebug_info\x18\x17 \x01(\x08:\x05\x66\x61lse\x12\"\n\x14snapshot_after_train\x18\x1c \x01(\x08:\x04true\x12;\n\x0bsolver_type\x18\x1e \x01(\x0e\x32!.caffe.SolverParameter.SolverType:\x03SGD\"+\n\x0eSnapshotFormat\x12\x08\n\x04HDF5\x10\x00\x12\x0f\n\x0b\x42INARYPROTO\x10\x01\"\x1e\n\nSolverMode\x12\x07\n\x03\x43PU\x10\x00\x12\x07\n\x03GPU\x10\x01\"U\n\nSolverType\x12\x07\n\x03SGD\x10\x00\x12\x0c\n\x08NESTEROV\x10\x01\x12\x0b\n\x07\x41\x44\x41GRAD\x10\x02\x12\x0b\n\x07RMSPROP\x10\x03\x12\x0c\n\x08\x41\x44\x41\x44\x45LTA\x10\x04\x12\x08\n\x04\x41\x44\x41M\x10\x05\"l\n\x0bSolverState\x12\x0c\n\x04iter\x18\x01 \x01(\x05\x12\x13\n\x0blearned_net\x18\x02 \x01(\t\x12!\n\x07history\x18\x03 \x03(\x0b\x32\x10.caffe.BlobProto\x12\x17\n\x0c\x63urrent_step\x18\x04 \x01(\x05:\x01\x30\"N\n\x08NetState\x12!\n\x05phase\x18\x01 \x01(\x0e\x32\x0c.caffe.Phase:\x04TEST\x12\x10\n\x05level\x18\x02 \x01(\x05:\x01\x30\x12\r\n\x05stage\x18\x03 \x03(\t\"s\n\x0cNetStateRule\x12\x1b\n\x05phase\x18\x01 \x01(\x0e\x32\x0c.caffe.Phase\x12\x11\n\tmin_level\x18\x02 \x01(\x05\x12\x11\n\tmax_level\x18\x03 \x01(\x05\x12\r\n\x05stage\x18\x04 \x03(\t\x12\x11\n\tnot_stage\x18\x05 \x03(\t\"\xa3\x01\n\tParamSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\nshare_mode\x18\x02 \x01(\x0e\x32\x1d.caffe.ParamSpec.DimCheckMode\x12\x12\n\x07lr_mult\x18\x03 \x01(\x02:\x01\x31\x12\x15\n\ndecay_mult\x18\x04 \x01(\x02:\x01\x31\"*\n\x0c\x44imCheckMode\x12\n\n\x06STRICT\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01\"\x98\x13\n\x0eLayerParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x62ottom\x18\x03 \x03(\t\x12\x0b\n\x03top\x18\x04 \x03(\t\x12\x1b\n\x05phase\x18\n \x01(\x0e\x32\x0c.caffe.Phase\x12\x13\n\x0bloss_weight\x18\x05 \x03(\x02\x12\x1f\n\x05param\x18\x06 \x03(\x0b\x32\x10.caffe.ParamSpec\x12\x1f\n\x05\x62lobs\x18\x07 \x03(\x0b\x32\x10.caffe.BlobProto\x12\x16\n\x0epropagate_down\x18\x0b \x03(\x08\x12$\n\x07include\x18\x08 \x03(\x0b\x32\x13.caffe.NetStateRule\x12$\n\x07\x65xclude\x18\t \x03(\x0b\x32\x13.caffe.NetStateRule\x12\x37\n\x0ftransform_param\x18\x64 \x01(\x0b\x32\x1e.caffe.TransformationParameter\x12(\n\nloss_param\x18\x65 \x01(\x0b\x32\x14.caffe.LossParameter\x12\x30\n\x0e\x61\x63\x63uracy_param\x18\x66 \x01(\x0b\x32\x18.caffe.AccuracyParameter\x12,\n\x0c\x61rgmax_param\x18g \x01(\x0b\x32\x16.caffe.ArgMaxParameter\x12\x34\n\x10\x62\x61tch_norm_param\x18\x8b\x01 \x01(\x0b\x32\x19.caffe.BatchNormParameter\x12)\n\nbias_param\x18\x8d\x01 \x01(\x0b\x32\x14.caffe.BiasParameter\x12,\n\x0c\x63oncat_param\x18h \x01(\x0b\x32\x16.caffe.ConcatParameter\x12?\n\x16\x63ontrastive_loss_param\x18i \x01(\x0b\x32\x1f.caffe.ContrastiveLossParameter\x12\x36\n\x11\x63onvolution_param\x18j \x01(\x0b\x32\x1b.caffe.ConvolutionParameter\x12)\n\ncrop_param\x18\x90\x01 \x01(\x0b\x32\x14.caffe.CropParameter\x12(\n\ndata_param\x18k \x01(\x0b\x32\x14.caffe.DataParameter\x12.\n\rdropout_param\x18l \x01(\x0b\x32\x17.caffe.DropoutParameter\x12\x33\n\x10\x64ummy_data_param\x18m \x01(\x0b\x32\x19.caffe.DummyDataParameter\x12.\n\reltwise_param\x18n \x01(\x0b\x32\x17.caffe.EltwiseParameter\x12\'\n\telu_param\x18\x8c\x01 \x01(\x0b\x32\x13.caffe.ELUParameter\x12+\n\x0b\x65mbed_param\x18\x89\x01 \x01(\x0b\x32\x15.caffe.EmbedParameter\x12&\n\texp_param\x18o \x01(\x0b\x32\x13.caffe.ExpParameter\x12/\n\rflatten_param\x18\x87\x01 \x01(\x0b\x32\x17.caffe.FlattenParameter\x12\x31\n\x0fhdf5_data_param\x18p \x01(\x0b\x32\x18.caffe.HDF5DataParameter\x12\x35\n\x11hdf5_output_param\x18q \x01(\x0b\x32\x1a.caffe.HDF5OutputParameter\x12\x33\n\x10hinge_loss_param\x18r \x01(\x0b\x32\x19.caffe.HingeLossParameter\x12\x33\n\x10image_data_param\x18s \x01(\x0b\x32\x19.caffe.ImageDataParameter\x12\x39\n\x13infogain_loss_param\x18t \x01(\x0b\x32\x1c.caffe.InfogainLossParameter\x12\x39\n\x13inner_product_param\x18u \x01(\x0b\x32\x1c.caffe.InnerProductParameter\x12+\n\x0binput_param\x18\x8f\x01 \x01(\x0b\x32\x15.caffe.InputParameter\x12\'\n\tlog_param\x18\x86\x01 \x01(\x0b\x32\x13.caffe.LogParameter\x12&\n\tlrn_param\x18v \x01(\x0b\x32\x13.caffe.LRNParameter\x12\x35\n\x11memory_data_param\x18w \x01(\x0b\x32\x1a.caffe.MemoryDataParameter\x12&\n\tmvn_param\x18x \x01(\x0b\x32\x13.caffe.MVNParameter\x12.\n\rpooling_param\x18y \x01(\x0b\x32\x17.caffe.PoolingParameter\x12*\n\x0bpower_param\x18z \x01(\x0b\x32\x15.caffe.PowerParameter\x12+\n\x0bprelu_param\x18\x83\x01 \x01(\x0b\x32\x15.caffe.PReLUParameter\x12-\n\x0cpython_param\x18\x82\x01 \x01(\x0b\x32\x16.caffe.PythonParameter\x12\x33\n\x0freduction_param\x18\x88\x01 \x01(\x0b\x32\x19.caffe.ReductionParameter\x12(\n\nrelu_param\x18{ \x01(\x0b\x32\x14.caffe.ReLUParameter\x12/\n\rreshape_param\x18\x85\x01 \x01(\x0b\x32\x17.caffe.ReshapeParameter\x12+\n\x0bscale_param\x18\x8e\x01 \x01(\x0b\x32\x15.caffe.ScaleParameter\x12.\n\rsigmoid_param\x18| \x01(\x0b\x32\x17.caffe.SigmoidParameter\x12.\n\rsoftmax_param\x18} \x01(\x0b\x32\x17.caffe.SoftmaxParameter\x12\'\n\tspp_param\x18\x84\x01 \x01(\x0b\x32\x13.caffe.SPPParameter\x12*\n\x0bslice_param\x18~ \x01(\x0b\x32\x15.caffe.SliceParameter\x12(\n\ntanh_param\x18\x7f \x01(\x0b\x32\x14.caffe.TanHParameter\x12\x33\n\x0fthreshold_param\x18\x80\x01 \x01(\x0b\x32\x19.caffe.ThresholdParameter\x12)\n\ntile_param\x18\x8a\x01 \x01(\x0b\x32\x14.caffe.TileParameter\x12\x36\n\x11window_data_param\x18\x81\x01 \x01(\x0b\x32\x1a.caffe.WindowDataParameter\"\xb6\x01\n\x17TransformationParameter\x12\x10\n\x05scale\x18\x01 \x01(\x02:\x01\x31\x12\x15\n\x06mirror\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x14\n\tcrop_size\x18\x03 \x01(\r:\x01\x30\x12\x11\n\tmean_file\x18\x04 \x01(\t\x12\x12\n\nmean_value\x18\x05 \x03(\x02\x12\x1a\n\x0b\x66orce_color\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\x19\n\nforce_gray\x18\x07 \x01(\x08:\x05\x66\x61lse\"\xc2\x01\n\rLossParameter\x12\x14\n\x0cignore_label\x18\x01 \x01(\x05\x12\x44\n\rnormalization\x18\x03 \x01(\x0e\x32&.caffe.LossParameter.NormalizationMode:\x05VALID\x12\x11\n\tnormalize\x18\x02 \x01(\x08\"B\n\x11NormalizationMode\x12\x08\n\x04\x46ULL\x10\x00\x12\t\n\x05VALID\x10\x01\x12\x0e\n\nBATCH_SIZE\x10\x02\x12\x08\n\x04NONE\x10\x03\"L\n\x11\x41\x63\x63uracyParameter\x12\x10\n\x05top_k\x18\x01 \x01(\r:\x01\x31\x12\x0f\n\x04\x61xis\x18\x02 \x01(\x05:\x01\x31\x12\x14\n\x0cignore_label\x18\x03 \x01(\x05\"M\n\x0f\x41rgMaxParameter\x12\x1a\n\x0bout_max_val\x18\x01 \x01(\x08:\x05\x66\x61lse\x12\x10\n\x05top_k\x18\x02 \x01(\r:\x01\x31\x12\x0c\n\x04\x61xis\x18\x03 \x01(\x05\"9\n\x0f\x43oncatParameter\x12\x0f\n\x04\x61xis\x18\x02 \x01(\x05:\x01\x31\x12\x15\n\nconcat_dim\x18\x01 \x01(\r:\x01\x31\"j\n\x12\x42\x61tchNormParameter\x12\x18\n\x10use_global_stats\x18\x01 \x01(\x08\x12&\n\x17moving_average_fraction\x18\x02 \x01(\x02:\x05\x30.999\x12\x12\n\x03\x65ps\x18\x03 \x01(\x02:\x05\x31\x65-05\"]\n\rBiasParameter\x12\x0f\n\x04\x61xis\x18\x01 \x01(\x05:\x01\x31\x12\x13\n\x08num_axes\x18\x02 \x01(\x05:\x01\x31\x12&\n\x06\x66iller\x18\x03 \x01(\x0b\x32\x16.caffe.FillerParameter\"L\n\x18\x43ontrastiveLossParameter\x12\x11\n\x06margin\x18\x01 \x01(\x02:\x01\x31\x12\x1d\n\x0elegacy_version\x18\x02 \x01(\x08:\x05\x66\x61lse\"\xfc\x03\n\x14\x43onvolutionParameter\x12\x12\n\nnum_output\x18\x01 \x01(\r\x12\x17\n\tbias_term\x18\x02 \x01(\x08:\x04true\x12\x0b\n\x03pad\x18\x03 \x03(\r\x12\x13\n\x0bkernel_size\x18\x04 \x03(\r\x12\x0e\n\x06stride\x18\x06 \x03(\r\x12\x10\n\x08\x64ilation\x18\x12 \x03(\r\x12\x10\n\x05pad_h\x18\t \x01(\r:\x01\x30\x12\x10\n\x05pad_w\x18\n \x01(\r:\x01\x30\x12\x10\n\x08kernel_h\x18\x0b \x01(\r\x12\x10\n\x08kernel_w\x18\x0c \x01(\r\x12\x10\n\x08stride_h\x18\r \x01(\r\x12\x10\n\x08stride_w\x18\x0e \x01(\r\x12\x10\n\x05group\x18\x05 \x01(\r:\x01\x31\x12-\n\rweight_filler\x18\x07 \x01(\x0b\x32\x16.caffe.FillerParameter\x12+\n\x0b\x62ias_filler\x18\x08 \x01(\x0b\x32\x16.caffe.FillerParameter\x12;\n\x06\x65ngine\x18\x0f \x01(\x0e\x32\".caffe.ConvolutionParameter.Engine:\x07\x44\x45\x46\x41ULT\x12\x0f\n\x04\x61xis\x18\x10 \x01(\x05:\x01\x31\x12\x1e\n\x0f\x66orce_nd_im2col\x18\x11 \x01(\x08:\x05\x66\x61lse\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"0\n\rCropParameter\x12\x0f\n\x04\x61xis\x18\x01 \x01(\x05:\x01\x32\x12\x0e\n\x06offset\x18\x02 \x03(\r\"\xa4\x02\n\rDataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x12\n\nbatch_size\x18\x04 \x01(\r\x12\x14\n\trand_skip\x18\x07 \x01(\r:\x01\x30\x12\x31\n\x07\x62\x61\x63kend\x18\x08 \x01(\x0e\x32\x17.caffe.DataParameter.DB:\x07LEVELDB\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x01\x31\x12\x11\n\tmean_file\x18\x03 \x01(\t\x12\x14\n\tcrop_size\x18\x05 \x01(\r:\x01\x30\x12\x15\n\x06mirror\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\"\n\x13\x66orce_encoded_color\x18\t \x01(\x08:\x05\x66\x61lse\x12\x13\n\x08prefetch\x18\n \x01(\r:\x01\x34\"\x1b\n\x02\x44\x42\x12\x0b\n\x07LEVELDB\x10\x00\x12\x08\n\x04LMDB\x10\x01\".\n\x10\x44ropoutParameter\x12\x1a\n\rdropout_ratio\x18\x01 \x01(\x02:\x03\x30.5\"\xa0\x01\n\x12\x44ummyDataParameter\x12+\n\x0b\x64\x61ta_filler\x18\x01 \x03(\x0b\x32\x16.caffe.FillerParameter\x12\x1f\n\x05shape\x18\x06 \x03(\x0b\x32\x10.caffe.BlobShape\x12\x0b\n\x03num\x18\x02 \x03(\r\x12\x10\n\x08\x63hannels\x18\x03 \x03(\r\x12\x0e\n\x06height\x18\x04 \x03(\r\x12\r\n\x05width\x18\x05 \x03(\r\"\xa5\x01\n\x10\x45ltwiseParameter\x12\x39\n\toperation\x18\x01 \x01(\x0e\x32!.caffe.EltwiseParameter.EltwiseOp:\x03SUM\x12\r\n\x05\x63oeff\x18\x02 \x03(\x02\x12\x1e\n\x10stable_prod_grad\x18\x03 \x01(\x08:\x04true\"\'\n\tEltwiseOp\x12\x08\n\x04PROD\x10\x00\x12\x07\n\x03SUM\x10\x01\x12\x07\n\x03MAX\x10\x02\" \n\x0c\x45LUParameter\x12\x10\n\x05\x61lpha\x18\x01 \x01(\x02:\x01\x31\"\xac\x01\n\x0e\x45mbedParameter\x12\x12\n\nnum_output\x18\x01 \x01(\r\x12\x11\n\tinput_dim\x18\x02 \x01(\r\x12\x17\n\tbias_term\x18\x03 \x01(\x08:\x04true\x12-\n\rweight_filler\x18\x04 \x01(\x0b\x32\x16.caffe.FillerParameter\x12+\n\x0b\x62ias_filler\x18\x05 \x01(\x0b\x32\x16.caffe.FillerParameter\"D\n\x0c\x45xpParameter\x12\x10\n\x04\x62\x61se\x18\x01 \x01(\x02:\x02-1\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x01\x31\x12\x10\n\x05shift\x18\x03 \x01(\x02:\x01\x30\"9\n\x10\x46lattenParameter\x12\x0f\n\x04\x61xis\x18\x01 \x01(\x05:\x01\x31\x12\x14\n\x08\x65nd_axis\x18\x02 \x01(\x05:\x02-1\"O\n\x11HDF5DataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x12\n\nbatch_size\x18\x02 \x01(\r\x12\x16\n\x07shuffle\x18\x03 \x01(\x08:\x05\x66\x61lse\"(\n\x13HDF5OutputParameter\x12\x11\n\tfile_name\x18\x01 \x01(\t\"^\n\x12HingeLossParameter\x12\x30\n\x04norm\x18\x01 \x01(\x0e\x32\x1e.caffe.HingeLossParameter.Norm:\x02L1\"\x16\n\x04Norm\x12\x06\n\x02L1\x10\x01\x12\x06\n\x02L2\x10\x02\"\x97\x02\n\x12ImageDataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x15\n\nbatch_size\x18\x04 \x01(\r:\x01\x31\x12\x14\n\trand_skip\x18\x07 \x01(\r:\x01\x30\x12\x16\n\x07shuffle\x18\x08 \x01(\x08:\x05\x66\x61lse\x12\x15\n\nnew_height\x18\t \x01(\r:\x01\x30\x12\x14\n\tnew_width\x18\n \x01(\r:\x01\x30\x12\x16\n\x08is_color\x18\x0b \x01(\x08:\x04true\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x01\x31\x12\x11\n\tmean_file\x18\x03 \x01(\t\x12\x14\n\tcrop_size\x18\x05 \x01(\r:\x01\x30\x12\x15\n\x06mirror\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\x15\n\x0broot_folder\x18\x0c \x01(\t:\x00\"\'\n\x15InfogainLossParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\"\xcb\x01\n\x15InnerProductParameter\x12\x12\n\nnum_output\x18\x01 \x01(\r\x12\x17\n\tbias_term\x18\x02 \x01(\x08:\x04true\x12-\n\rweight_filler\x18\x03 \x01(\x0b\x32\x16.caffe.FillerParameter\x12+\n\x0b\x62ias_filler\x18\x04 \x01(\x0b\x32\x16.caffe.FillerParameter\x12\x0f\n\x04\x61xis\x18\x05 \x01(\x05:\x01\x31\x12\x18\n\ttranspose\x18\x06 \x01(\x08:\x05\x66\x61lse\"1\n\x0eInputParameter\x12\x1f\n\x05shape\x18\x01 \x03(\x0b\x32\x10.caffe.BlobShape\"D\n\x0cLogParameter\x12\x10\n\x04\x62\x61se\x18\x01 \x01(\x02:\x02-1\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x01\x31\x12\x10\n\x05shift\x18\x03 \x01(\x02:\x01\x30\"\xb8\x02\n\x0cLRNParameter\x12\x15\n\nlocal_size\x18\x01 \x01(\r:\x01\x35\x12\x10\n\x05\x61lpha\x18\x02 \x01(\x02:\x01\x31\x12\x12\n\x04\x62\x65ta\x18\x03 \x01(\x02:\x04\x30.75\x12\x44\n\x0bnorm_region\x18\x04 \x01(\x0e\x32\x1e.caffe.LRNParameter.NormRegion:\x0f\x41\x43ROSS_CHANNELS\x12\x0c\n\x01k\x18\x05 \x01(\x02:\x01\x31\x12\x33\n\x06\x65ngine\x18\x06 \x01(\x0e\x32\x1a.caffe.LRNParameter.Engine:\x07\x44\x45\x46\x41ULT\"5\n\nNormRegion\x12\x13\n\x0f\x41\x43ROSS_CHANNELS\x10\x00\x12\x12\n\x0eWITHIN_CHANNEL\x10\x01\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"Z\n\x13MemoryDataParameter\x12\x12\n\nbatch_size\x18\x01 \x01(\r\x12\x10\n\x08\x63hannels\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\r\n\x05width\x18\x04 \x01(\r\"d\n\x0cMVNParameter\x12 \n\x12normalize_variance\x18\x01 \x01(\x08:\x04true\x12\x1e\n\x0f\x61\x63ross_channels\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x12\n\x03\x65ps\x18\x03 \x01(\x02:\x05\x31\x65-09\"\xa2\x03\n\x10PoolingParameter\x12\x35\n\x04pool\x18\x01 \x01(\x0e\x32\".caffe.PoolingParameter.PoolMethod:\x03MAX\x12\x0e\n\x03pad\x18\x04 \x01(\r:\x01\x30\x12\x10\n\x05pad_h\x18\t \x01(\r:\x01\x30\x12\x10\n\x05pad_w\x18\n \x01(\r:\x01\x30\x12\x13\n\x0bkernel_size\x18\x02 \x01(\r\x12\x10\n\x08kernel_h\x18\x05 \x01(\r\x12\x10\n\x08kernel_w\x18\x06 \x01(\r\x12\x11\n\x06stride\x18\x03 \x01(\r:\x01\x31\x12\x10\n\x08stride_h\x18\x07 \x01(\r\x12\x10\n\x08stride_w\x18\x08 \x01(\r\x12\x37\n\x06\x65ngine\x18\x0b \x01(\x0e\x32\x1e.caffe.PoolingParameter.Engine:\x07\x44\x45\x46\x41ULT\x12\x1d\n\x0eglobal_pooling\x18\x0c \x01(\x08:\x05\x66\x61lse\".\n\nPoolMethod\x12\x07\n\x03MAX\x10\x00\x12\x07\n\x03\x41VE\x10\x01\x12\x0e\n\nSTOCHASTIC\x10\x02\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"F\n\x0ePowerParameter\x12\x10\n\x05power\x18\x01 \x01(\x02:\x01\x31\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x01\x31\x12\x10\n\x05shift\x18\x03 \x01(\x02:\x01\x30\"g\n\x0fPythonParameter\x12\x0e\n\x06module\x18\x01 \x01(\t\x12\r\n\x05layer\x18\x02 \x01(\t\x12\x13\n\tparam_str\x18\x03 \x01(\t:\x00\x12 \n\x11share_in_parallel\x18\x04 \x01(\x08:\x05\x66\x61lse\"\xad\x01\n\x12ReductionParameter\x12=\n\toperation\x18\x01 \x01(\x0e\x32%.caffe.ReductionParameter.ReductionOp:\x03SUM\x12\x0f\n\x04\x61xis\x18\x02 \x01(\x05:\x01\x30\x12\x10\n\x05\x63oeff\x18\x03 \x01(\x02:\x01\x31\"5\n\x0bReductionOp\x12\x07\n\x03SUM\x10\x01\x12\x08\n\x04\x41SUM\x10\x02\x12\t\n\x05SUMSQ\x10\x03\x12\x08\n\x04MEAN\x10\x04\"\x8d\x01\n\rReLUParameter\x12\x19\n\x0enegative_slope\x18\x01 \x01(\x02:\x01\x30\x12\x34\n\x06\x65ngine\x18\x02 \x01(\x0e\x32\x1b.caffe.ReLUParameter.Engine:\x07\x44\x45\x46\x41ULT\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"Z\n\x10ReshapeParameter\x12\x1f\n\x05shape\x18\x01 \x01(\x0b\x32\x10.caffe.BlobShape\x12\x0f\n\x04\x61xis\x18\x02 \x01(\x05:\x01\x30\x12\x14\n\x08num_axes\x18\x03 \x01(\x05:\x02-1\"\xa5\x01\n\x0eScaleParameter\x12\x0f\n\x04\x61xis\x18\x01 \x01(\x05:\x01\x31\x12\x13\n\x08num_axes\x18\x02 \x01(\x05:\x01\x31\x12&\n\x06\x66iller\x18\x03 \x01(\x0b\x32\x16.caffe.FillerParameter\x12\x18\n\tbias_term\x18\x04 \x01(\x08:\x05\x66\x61lse\x12+\n\x0b\x62ias_filler\x18\x05 \x01(\x0b\x32\x16.caffe.FillerParameter\"x\n\x10SigmoidParameter\x12\x37\n\x06\x65ngine\x18\x01 \x01(\x0e\x32\x1e.caffe.SigmoidParameter.Engine:\x07\x44\x45\x46\x41ULT\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"L\n\x0eSliceParameter\x12\x0f\n\x04\x61xis\x18\x03 \x01(\x05:\x01\x31\x12\x13\n\x0bslice_point\x18\x02 \x03(\r\x12\x14\n\tslice_dim\x18\x01 \x01(\r:\x01\x31\"\x89\x01\n\x10SoftmaxParameter\x12\x37\n\x06\x65ngine\x18\x01 \x01(\x0e\x32\x1e.caffe.SoftmaxParameter.Engine:\x07\x44\x45\x46\x41ULT\x12\x0f\n\x04\x61xis\x18\x02 \x01(\x05:\x01\x31\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"r\n\rTanHParameter\x12\x34\n\x06\x65ngine\x18\x01 \x01(\x0e\x32\x1b.caffe.TanHParameter.Engine:\x07\x44\x45\x46\x41ULT\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"/\n\rTileParameter\x12\x0f\n\x04\x61xis\x18\x01 \x01(\x05:\x01\x31\x12\r\n\x05tiles\x18\x02 \x01(\x05\"*\n\x12ThresholdParameter\x12\x14\n\tthreshold\x18\x01 \x01(\x02:\x01\x30\"\xc1\x02\n\x13WindowDataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x01\x31\x12\x11\n\tmean_file\x18\x03 \x01(\t\x12\x12\n\nbatch_size\x18\x04 \x01(\r\x12\x14\n\tcrop_size\x18\x05 \x01(\r:\x01\x30\x12\x15\n\x06mirror\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\x19\n\x0c\x66g_threshold\x18\x07 \x01(\x02:\x03\x30.5\x12\x19\n\x0c\x62g_threshold\x18\x08 \x01(\x02:\x03\x30.5\x12\x19\n\x0b\x66g_fraction\x18\t \x01(\x02:\x04\x30.25\x12\x16\n\x0b\x63ontext_pad\x18\n \x01(\r:\x01\x30\x12\x17\n\tcrop_mode\x18\x0b \x01(\t:\x04warp\x12\x1b\n\x0c\x63\x61\x63he_images\x18\x0c \x01(\x08:\x05\x66\x61lse\x12\x15\n\x0broot_folder\x18\r \x01(\t:\x00\"\xeb\x01\n\x0cSPPParameter\x12\x16\n\x0epyramid_height\x18\x01 \x01(\r\x12\x31\n\x04pool\x18\x02 \x01(\x0e\x32\x1e.caffe.SPPParameter.PoolMethod:\x03MAX\x12\x33\n\x06\x65ngine\x18\x06 \x01(\x0e\x32\x1a.caffe.SPPParameter.Engine:\x07\x44\x45\x46\x41ULT\".\n\nPoolMethod\x12\x07\n\x03MAX\x10\x00\x12\x07\n\x03\x41VE\x10\x01\x12\x0e\n\nSTOCHASTIC\x10\x02\"+\n\x06\x45ngine\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\t\n\x05\x43\x41\x46\x46\x45\x10\x01\x12\t\n\x05\x43UDNN\x10\x02\"\xe0\x13\n\x10V1LayerParameter\x12\x0e\n\x06\x62ottom\x18\x02 \x03(\t\x12\x0b\n\x03top\x18\x03 \x03(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12$\n\x07include\x18 \x03(\x0b\x32\x13.caffe.NetStateRule\x12$\n\x07\x65xclude\x18! \x03(\x0b\x32\x13.caffe.NetStateRule\x12/\n\x04type\x18\x05 \x01(\x0e\x32!.caffe.V1LayerParameter.LayerType\x12\x1f\n\x05\x62lobs\x18\x06 \x03(\x0b\x32\x10.caffe.BlobProto\x12\x0e\n\x05param\x18\xe9\x07 \x03(\t\x12>\n\x0f\x62lob_share_mode\x18\xea\x07 \x03(\x0e\x32$.caffe.V1LayerParameter.DimCheckMode\x12\x10\n\x08\x62lobs_lr\x18\x07 \x03(\x02\x12\x14\n\x0cweight_decay\x18\x08 \x03(\x02\x12\x13\n\x0bloss_weight\x18# \x03(\x02\x12\x30\n\x0e\x61\x63\x63uracy_param\x18\x1b \x01(\x0b\x32\x18.caffe.AccuracyParameter\x12,\n\x0c\x61rgmax_param\x18\x17 \x01(\x0b\x32\x16.caffe.ArgMaxParameter\x12,\n\x0c\x63oncat_param\x18\t \x01(\x0b\x32\x16.caffe.ConcatParameter\x12?\n\x16\x63ontrastive_loss_param\x18( \x01(\x0b\x32\x1f.caffe.ContrastiveLossParameter\x12\x36\n\x11\x63onvolution_param\x18\n \x01(\x0b\x32\x1b.caffe.ConvolutionParameter\x12(\n\ndata_param\x18\x0b \x01(\x0b\x32\x14.caffe.DataParameter\x12.\n\rdropout_param\x18\x0c \x01(\x0b\x32\x17.caffe.DropoutParameter\x12\x33\n\x10\x64ummy_data_param\x18\x1a \x01(\x0b\x32\x19.caffe.DummyDataParameter\x12.\n\reltwise_param\x18\x18 \x01(\x0b\x32\x17.caffe.EltwiseParameter\x12&\n\texp_param\x18) \x01(\x0b\x32\x13.caffe.ExpParameter\x12\x31\n\x0fhdf5_data_param\x18\r \x01(\x0b\x32\x18.caffe.HDF5DataParameter\x12\x35\n\x11hdf5_output_param\x18\x0e \x01(\x0b\x32\x1a.caffe.HDF5OutputParameter\x12\x33\n\x10hinge_loss_param\x18\x1d \x01(\x0b\x32\x19.caffe.HingeLossParameter\x12\x33\n\x10image_data_param\x18\x0f \x01(\x0b\x32\x19.caffe.ImageDataParameter\x12\x39\n\x13infogain_loss_param\x18\x10 \x01(\x0b\x32\x1c.caffe.InfogainLossParameter\x12\x39\n\x13inner_product_param\x18\x11 \x01(\x0b\x32\x1c.caffe.InnerProductParameter\x12&\n\tlrn_param\x18\x12 \x01(\x0b\x32\x13.caffe.LRNParameter\x12\x35\n\x11memory_data_param\x18\x16 \x01(\x0b\x32\x1a.caffe.MemoryDataParameter\x12&\n\tmvn_param\x18\" \x01(\x0b\x32\x13.caffe.MVNParameter\x12.\n\rpooling_param\x18\x13 \x01(\x0b\x32\x17.caffe.PoolingParameter\x12*\n\x0bpower_param\x18\x15 \x01(\x0b\x32\x15.caffe.PowerParameter\x12(\n\nrelu_param\x18\x1e \x01(\x0b\x32\x14.caffe.ReLUParameter\x12.\n\rsigmoid_param\x18& \x01(\x0b\x32\x17.caffe.SigmoidParameter\x12.\n\rsoftmax_param\x18\' \x01(\x0b\x32\x17.caffe.SoftmaxParameter\x12*\n\x0bslice_param\x18\x1f \x01(\x0b\x32\x15.caffe.SliceParameter\x12(\n\ntanh_param\x18% \x01(\x0b\x32\x14.caffe.TanHParameter\x12\x32\n\x0fthreshold_param\x18\x19 \x01(\x0b\x32\x19.caffe.ThresholdParameter\x12\x35\n\x11window_data_param\x18\x14 \x01(\x0b\x32\x1a.caffe.WindowDataParameter\x12\x37\n\x0ftransform_param\x18$ \x01(\x0b\x32\x1e.caffe.TransformationParameter\x12(\n\nloss_param\x18* \x01(\x0b\x32\x14.caffe.LossParameter\x12&\n\x05layer\x18\x01 \x01(\x0b\x32\x17.caffe.V0LayerParameter\"\xd8\x04\n\tLayerType\x12\x08\n\x04NONE\x10\x00\x12\n\n\x06\x41\x42SVAL\x10#\x12\x0c\n\x08\x41\x43\x43URACY\x10\x01\x12\n\n\x06\x41RGMAX\x10\x1e\x12\x08\n\x04\x42NLL\x10\x02\x12\n\n\x06\x43ONCAT\x10\x03\x12\x14\n\x10\x43ONTRASTIVE_LOSS\x10%\x12\x0f\n\x0b\x43ONVOLUTION\x10\x04\x12\x08\n\x04\x44\x41TA\x10\x05\x12\x11\n\rDECONVOLUTION\x10\'\x12\x0b\n\x07\x44ROPOUT\x10\x06\x12\x0e\n\nDUMMY_DATA\x10 \x12\x12\n\x0e\x45UCLIDEAN_LOSS\x10\x07\x12\x0b\n\x07\x45LTWISE\x10\x19\x12\x07\n\x03\x45XP\x10&\x12\x0b\n\x07\x46LATTEN\x10\x08\x12\r\n\tHDF5_DATA\x10\t\x12\x0f\n\x0bHDF5_OUTPUT\x10\n\x12\x0e\n\nHINGE_LOSS\x10\x1c\x12\n\n\x06IM2COL\x10\x0b\x12\x0e\n\nIMAGE_DATA\x10\x0c\x12\x11\n\rINFOGAIN_LOSS\x10\r\x12\x11\n\rINNER_PRODUCT\x10\x0e\x12\x07\n\x03LRN\x10\x0f\x12\x0f\n\x0bMEMORY_DATA\x10\x1d\x12\x1d\n\x19MULTINOMIAL_LOGISTIC_LOSS\x10\x10\x12\x07\n\x03MVN\x10\"\x12\x0b\n\x07POOLING\x10\x11\x12\t\n\x05POWER\x10\x1a\x12\x08\n\x04RELU\x10\x12\x12\x0b\n\x07SIGMOID\x10\x13\x12\x1e\n\x1aSIGMOID_CROSS_ENTROPY_LOSS\x10\x1b\x12\x0b\n\x07SILENCE\x10$\x12\x0b\n\x07SOFTMAX\x10\x14\x12\x10\n\x0cSOFTMAX_LOSS\x10\x15\x12\t\n\x05SPLIT\x10\x16\x12\t\n\x05SLICE\x10!\x12\x08\n\x04TANH\x10\x17\x12\x0f\n\x0bWINDOW_DATA\x10\x18\x12\r\n\tTHRESHOLD\x10\x1f\"*\n\x0c\x44imCheckMode\x12\n\n\x06STRICT\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01\"\xfd\x07\n\x10V0LayerParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x12\n\nnum_output\x18\x03 \x01(\r\x12\x16\n\x08\x62iasterm\x18\x04 \x01(\x08:\x04true\x12-\n\rweight_filler\x18\x05 \x01(\x0b\x32\x16.caffe.FillerParameter\x12+\n\x0b\x62ias_filler\x18\x06 \x01(\x0b\x32\x16.caffe.FillerParameter\x12\x0e\n\x03pad\x18\x07 \x01(\r:\x01\x30\x12\x12\n\nkernelsize\x18\x08 \x01(\r\x12\x10\n\x05group\x18\t \x01(\r:\x01\x31\x12\x11\n\x06stride\x18\n \x01(\r:\x01\x31\x12\x35\n\x04pool\x18\x0b \x01(\x0e\x32\".caffe.V0LayerParameter.PoolMethod:\x03MAX\x12\x1a\n\rdropout_ratio\x18\x0c \x01(\x02:\x03\x30.5\x12\x15\n\nlocal_size\x18\r \x01(\r:\x01\x35\x12\x10\n\x05\x61lpha\x18\x0e \x01(\x02:\x01\x31\x12\x12\n\x04\x62\x65ta\x18\x0f \x01(\x02:\x04\x30.75\x12\x0c\n\x01k\x18\x16 \x01(\x02:\x01\x31\x12\x0e\n\x06source\x18\x10 \x01(\t\x12\x10\n\x05scale\x18\x11 \x01(\x02:\x01\x31\x12\x10\n\x08meanfile\x18\x12 \x01(\t\x12\x11\n\tbatchsize\x18\x13 \x01(\r\x12\x13\n\x08\x63ropsize\x18\x14 \x01(\r:\x01\x30\x12\x15\n\x06mirror\x18\x15 \x01(\x08:\x05\x66\x61lse\x12\x1f\n\x05\x62lobs\x18\x32 \x03(\x0b\x32\x10.caffe.BlobProto\x12\x10\n\x08\x62lobs_lr\x18\x33 \x03(\x02\x12\x14\n\x0cweight_decay\x18\x34 \x03(\x02\x12\x14\n\trand_skip\x18\x35 \x01(\r:\x01\x30\x12\x1d\n\x10\x64\x65t_fg_threshold\x18\x36 \x01(\x02:\x03\x30.5\x12\x1d\n\x10\x64\x65t_bg_threshold\x18\x37 \x01(\x02:\x03\x30.5\x12\x1d\n\x0f\x64\x65t_fg_fraction\x18\x38 \x01(\x02:\x04\x30.25\x12\x1a\n\x0f\x64\x65t_context_pad\x18: \x01(\r:\x01\x30\x12\x1b\n\rdet_crop_mode\x18; \x01(\t:\x04warp\x12\x12\n\x07new_num\x18< \x01(\x05:\x01\x30\x12\x17\n\x0cnew_channels\x18= \x01(\x05:\x01\x30\x12\x15\n\nnew_height\x18> \x01(\x05:\x01\x30\x12\x14\n\tnew_width\x18? \x01(\x05:\x01\x30\x12\x1d\n\x0eshuffle_images\x18@ \x01(\x08:\x05\x66\x61lse\x12\x15\n\nconcat_dim\x18\x41 \x01(\r:\x01\x31\x12\x36\n\x11hdf5_output_param\x18\xe9\x07 \x01(\x0b\x32\x1a.caffe.HDF5OutputParameter\".\n\nPoolMethod\x12\x07\n\x03MAX\x10\x00\x12\x07\n\x03\x41VE\x10\x01\x12\x0e\n\nSTOCHASTIC\x10\x02\"W\n\x0ePReLUParameter\x12&\n\x06\x66iller\x18\x01 \x01(\x0b\x32\x16.caffe.FillerParameter\x12\x1d\n\x0e\x63hannel_shared\x18\x02 \x01(\x08:\x05\x66\x61lse*\x1c\n\x05Phase\x12\t\n\x05TRAIN\x10\x00\x12\x08\n\x04TEST\x10\x01') - -_PHASE = _descriptor.EnumDescriptor( - name='Phase', - full_name='caffe.Phase', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='TRAIN', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TEST', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=14991, - serialized_end=15019, -) - -Phase = enum_type_wrapper.EnumTypeWrapper(_PHASE) -TRAIN = 0 -TEST = 1 - - -_FILLERPARAMETER_VARIANCENORM = _descriptor.EnumDescriptor( - name='VarianceNorm', - full_name='caffe.FillerParameter.VarianceNorm', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='FAN_IN', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FAN_OUT', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AVERAGE', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=658, - serialized_end=710, -) - -_SOLVERPARAMETER_SNAPSHOTFORMAT = _descriptor.EnumDescriptor( - name='SnapshotFormat', - full_name='caffe.SolverParameter.SnapshotFormat', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='HDF5', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BINARYPROTO', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=2132, - serialized_end=2175, -) - -_SOLVERPARAMETER_SOLVERMODE = _descriptor.EnumDescriptor( - name='SolverMode', - full_name='caffe.SolverParameter.SolverMode', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='CPU', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GPU', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=2177, - serialized_end=2207, -) - -_SOLVERPARAMETER_SOLVERTYPE = _descriptor.EnumDescriptor( - name='SolverType', - full_name='caffe.SolverParameter.SolverType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='SGD', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NESTEROV', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ADAGRAD', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RMSPROP', index=3, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ADADELTA', index=4, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ADAM', index=5, number=5, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=2209, - serialized_end=2294, -) - -_PARAMSPEC_DIMCHECKMODE = _descriptor.EnumDescriptor( - name='DimCheckMode', - full_name='caffe.ParamSpec.DimCheckMode', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='STRICT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PERMISSIVE', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=2725, - serialized_end=2767, -) - -_LOSSPARAMETER_NORMALIZATIONMODE = _descriptor.EnumDescriptor( - name='NormalizationMode', - full_name='caffe.LossParameter.NormalizationMode', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='FULL', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VALID', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BATCH_SIZE', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NONE', index=3, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=5542, - serialized_end=5608, -) - -_CONVOLUTIONPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.ConvolutionParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_DATAPARAMETER_DB = _descriptor.EnumDescriptor( - name='DB', - full_name='caffe.DataParameter.DB', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='LEVELDB', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LMDB', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6934, - serialized_end=6961, -) - -_ELTWISEPARAMETER_ELTWISEOP = _descriptor.EnumDescriptor( - name='EltwiseOp', - full_name='caffe.EltwiseParameter.EltwiseOp', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='PROD', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SUM', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MAX', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=7301, - serialized_end=7340, -) - -_HINGELOSSPARAMETER_NORM = _descriptor.EnumDescriptor( - name='Norm', - full_name='caffe.HingeLossParameter.Norm', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='L1', index=0, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='L2', index=1, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=7875, - serialized_end=7897, -) - -_LRNPARAMETER_NORMREGION = _descriptor.EnumDescriptor( - name='NormRegion', - full_name='caffe.LRNParameter.NormRegion', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='ACROSS_CHANNELS', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WITHIN_CHANNEL', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=8764, - serialized_end=8817, -) - -_LRNPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.LRNParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_POOLINGPARAMETER_POOLMETHOD = _descriptor.EnumDescriptor( - name='PoolMethod', - full_name='caffe.PoolingParameter.PoolMethod', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='MAX', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AVE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='STOCHASTIC', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=9386, - serialized_end=9432, -) - -_POOLINGPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.PoolingParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_REDUCTIONPARAMETER_REDUCTIONOP = _descriptor.EnumDescriptor( - name='ReductionOp', - full_name='caffe.ReductionParameter.ReductionOp', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='SUM', index=0, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ASUM', index=1, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SUMSQ', index=2, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MEAN', index=3, number=4, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=9777, - serialized_end=9830, -) - -_RELUPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.ReLUParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_SIGMOIDPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.SigmoidParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_SOFTMAXPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.SoftmaxParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_TANHPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.TanHParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_SPPPARAMETER_POOLMETHOD = _descriptor.EnumDescriptor( - name='PoolMethod', - full_name='caffe.SPPParameter.PoolMethod', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='MAX', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AVE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='STOCHASTIC', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=9386, - serialized_end=9432, -) - -_SPPPARAMETER_ENGINE = _descriptor.EnumDescriptor( - name='Engine', - full_name='caffe.SPPParameter.Engine', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CAFFE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CUDNN', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=6573, - serialized_end=6616, -) - -_V1LAYERPARAMETER_LAYERTYPE = _descriptor.EnumDescriptor( - name='LayerType', - full_name='caffe.V1LayerParameter.LayerType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='NONE', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ABSVAL', index=1, number=35, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ACCURACY', index=2, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ARGMAX', index=3, number=30, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BNLL', index=4, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CONCAT', index=5, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CONTRASTIVE_LOSS', index=6, number=37, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CONVOLUTION', index=7, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DATA', index=8, number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DECONVOLUTION', index=9, number=39, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DROPOUT', index=10, number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DUMMY_DATA', index=11, number=32, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EUCLIDEAN_LOSS', index=12, number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ELTWISE', index=13, number=25, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EXP', index=14, number=38, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FLATTEN', index=15, number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HDF5_DATA', index=16, number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HDF5_OUTPUT', index=17, number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HINGE_LOSS', index=18, number=28, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IM2COL', index=19, number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IMAGE_DATA', index=20, number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='INFOGAIN_LOSS', index=21, number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='INNER_PRODUCT', index=22, number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LRN', index=23, number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MEMORY_DATA', index=24, number=29, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MULTINOMIAL_LOGISTIC_LOSS', index=25, number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MVN', index=26, number=34, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='POOLING', index=27, number=17, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='POWER', index=28, number=26, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RELU', index=29, number=18, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SIGMOID', index=30, number=19, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SIGMOID_CROSS_ENTROPY_LOSS', index=31, number=27, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SILENCE', index=32, number=36, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SOFTMAX', index=33, number=20, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SOFTMAX_LOSS', index=34, number=21, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SPLIT', index=35, number=22, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SLICE', index=36, number=33, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TANH', index=37, number=23, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WINDOW_DATA', index=38, number=24, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='THRESHOLD', index=39, number=31, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=13232, - serialized_end=13832, -) - -_V1LAYERPARAMETER_DIMCHECKMODE = _descriptor.EnumDescriptor( - name='DimCheckMode', - full_name='caffe.V1LayerParameter.DimCheckMode', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='STRICT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PERMISSIVE', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=2725, - serialized_end=2767, -) - -_V0LAYERPARAMETER_POOLMETHOD = _descriptor.EnumDescriptor( - name='PoolMethod', - full_name='caffe.V0LayerParameter.PoolMethod', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='MAX', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AVE', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='STOCHASTIC', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=9386, - serialized_end=9432, -) - - -_BLOBSHAPE = _descriptor.Descriptor( - name='BlobShape', - full_name='caffe.BlobShape', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='dim', full_name='caffe.BlobShape.dim', index=0, - number=1, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=22, - serialized_end=50, -) - - -_BLOBPROTO = _descriptor.Descriptor( - name='BlobProto', - full_name='caffe.BlobProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='shape', full_name='caffe.BlobProto.shape', index=0, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='data', full_name='caffe.BlobProto.data', index=1, - number=5, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - _descriptor.FieldDescriptor( - name='diff', full_name='caffe.BlobProto.diff', index=2, - number=6, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - _descriptor.FieldDescriptor( - name='double_data', full_name='caffe.BlobProto.double_data', index=3, - number=8, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - _descriptor.FieldDescriptor( - name='double_diff', full_name='caffe.BlobProto.double_diff', index=4, - number=9, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - _descriptor.FieldDescriptor( - name='num', full_name='caffe.BlobProto.num', index=5, - number=1, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='channels', full_name='caffe.BlobProto.channels', index=6, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='height', full_name='caffe.BlobProto.height', index=7, - number=3, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='width', full_name='caffe.BlobProto.width', index=8, - number=4, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=53, - serialized_end=257, -) - - -_BLOBPROTOVECTOR = _descriptor.Descriptor( - name='BlobProtoVector', - full_name='caffe.BlobProtoVector', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='blobs', full_name='caffe.BlobProtoVector.blobs', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=259, - serialized_end=309, -) - - -_DATUM = _descriptor.Descriptor( - name='Datum', - full_name='caffe.Datum', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='channels', full_name='caffe.Datum.channels', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='height', full_name='caffe.Datum.height', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='width', full_name='caffe.Datum.width', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='data', full_name='caffe.Datum.data', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='label', full_name='caffe.Datum.label', index=4, - number=5, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='float_data', full_name='caffe.Datum.float_data', index=5, - number=6, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='encoded', full_name='caffe.Datum.encoded', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=312, - serialized_end=441, -) - - -_FILLERPARAMETER = _descriptor.Descriptor( - name='FillerParameter', - full_name='caffe.FillerParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='type', full_name='caffe.FillerParameter.type', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("constant", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='caffe.FillerParameter.value', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='min', full_name='caffe.FillerParameter.min', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='max', full_name='caffe.FillerParameter.max', index=3, - number=4, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mean', full_name='caffe.FillerParameter.mean', index=4, - number=5, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='std', full_name='caffe.FillerParameter.std', index=5, - number=6, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sparse', full_name='caffe.FillerParameter.sparse', index=6, - number=7, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='variance_norm', full_name='caffe.FillerParameter.variance_norm', index=7, - number=8, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _FILLERPARAMETER_VARIANCENORM, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=444, - serialized_end=710, -) - - -_NETPARAMETER = _descriptor.Descriptor( - name='NetParameter', - full_name='caffe.NetParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='caffe.NetParameter.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='input', full_name='caffe.NetParameter.input', index=1, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='input_shape', full_name='caffe.NetParameter.input_shape', index=2, - number=8, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='input_dim', full_name='caffe.NetParameter.input_dim', index=3, - number=4, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='force_backward', full_name='caffe.NetParameter.force_backward', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='state', full_name='caffe.NetParameter.state', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='debug_info', full_name='caffe.NetParameter.debug_info', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='layer', full_name='caffe.NetParameter.layer', index=7, - number=100, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='layers', full_name='caffe.NetParameter.layers', index=8, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=713, - serialized_end=983, -) - - -_SOLVERPARAMETER = _descriptor.Descriptor( - name='SolverParameter', - full_name='caffe.SolverParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='net', full_name='caffe.SolverParameter.net', index=0, - number=24, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='net_param', full_name='caffe.SolverParameter.net_param', index=1, - number=25, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='train_net', full_name='caffe.SolverParameter.train_net', index=2, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_net', full_name='caffe.SolverParameter.test_net', index=3, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='train_net_param', full_name='caffe.SolverParameter.train_net_param', index=4, - number=21, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_net_param', full_name='caffe.SolverParameter.test_net_param', index=5, - number=22, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='train_state', full_name='caffe.SolverParameter.train_state', index=6, - number=26, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_state', full_name='caffe.SolverParameter.test_state', index=7, - number=27, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_iter', full_name='caffe.SolverParameter.test_iter', index=8, - number=3, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_interval', full_name='caffe.SolverParameter.test_interval', index=9, - number=4, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_compute_loss', full_name='caffe.SolverParameter.test_compute_loss', index=10, - number=19, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='test_initialization', full_name='caffe.SolverParameter.test_initialization', index=11, - number=32, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='base_lr', full_name='caffe.SolverParameter.base_lr', index=12, - number=5, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='display', full_name='caffe.SolverParameter.display', index=13, - number=6, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='average_loss', full_name='caffe.SolverParameter.average_loss', index=14, - number=33, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='max_iter', full_name='caffe.SolverParameter.max_iter', index=15, - number=7, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='iter_size', full_name='caffe.SolverParameter.iter_size', index=16, - number=36, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='lr_policy', full_name='caffe.SolverParameter.lr_policy', index=17, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='gamma', full_name='caffe.SolverParameter.gamma', index=18, - number=9, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='power', full_name='caffe.SolverParameter.power', index=19, - number=10, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='momentum', full_name='caffe.SolverParameter.momentum', index=20, - number=11, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_decay', full_name='caffe.SolverParameter.weight_decay', index=21, - number=12, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='regularization_type', full_name='caffe.SolverParameter.regularization_type', index=22, - number=29, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("L2", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stepsize', full_name='caffe.SolverParameter.stepsize', index=23, - number=13, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stepvalue', full_name='caffe.SolverParameter.stepvalue', index=24, - number=34, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='clip_gradients', full_name='caffe.SolverParameter.clip_gradients', index=25, - number=35, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='snapshot', full_name='caffe.SolverParameter.snapshot', index=26, - number=14, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='snapshot_prefix', full_name='caffe.SolverParameter.snapshot_prefix', index=27, - number=15, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='snapshot_diff', full_name='caffe.SolverParameter.snapshot_diff', index=28, - number=16, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='snapshot_format', full_name='caffe.SolverParameter.snapshot_format', index=29, - number=37, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='solver_mode', full_name='caffe.SolverParameter.solver_mode', index=30, - number=17, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='device_id', full_name='caffe.SolverParameter.device_id', index=31, - number=18, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='random_seed', full_name='caffe.SolverParameter.random_seed', index=32, - number=20, type=3, cpp_type=2, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='type', full_name='caffe.SolverParameter.type', index=33, - number=40, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("SGD", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delta', full_name='caffe.SolverParameter.delta', index=34, - number=31, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1e-08, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='momentum2', full_name='caffe.SolverParameter.momentum2', index=35, - number=39, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.999, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='rms_decay', full_name='caffe.SolverParameter.rms_decay', index=36, - number=38, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='debug_info', full_name='caffe.SolverParameter.debug_info', index=37, - number=23, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='snapshot_after_train', full_name='caffe.SolverParameter.snapshot_after_train', index=38, - number=28, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='solver_type', full_name='caffe.SolverParameter.solver_type', index=39, - number=30, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SOLVERPARAMETER_SNAPSHOTFORMAT, - _SOLVERPARAMETER_SOLVERMODE, - _SOLVERPARAMETER_SOLVERTYPE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=986, - serialized_end=2294, -) - - -_SOLVERSTATE = _descriptor.Descriptor( - name='SolverState', - full_name='caffe.SolverState', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='iter', full_name='caffe.SolverState.iter', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='learned_net', full_name='caffe.SolverState.learned_net', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='history', full_name='caffe.SolverState.history', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='current_step', full_name='caffe.SolverState.current_step', index=3, - number=4, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=2296, - serialized_end=2404, -) - - -_NETSTATE = _descriptor.Descriptor( - name='NetState', - full_name='caffe.NetState', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='phase', full_name='caffe.NetState.phase', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='level', full_name='caffe.NetState.level', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stage', full_name='caffe.NetState.stage', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=2406, - serialized_end=2484, -) - - -_NETSTATERULE = _descriptor.Descriptor( - name='NetStateRule', - full_name='caffe.NetStateRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='phase', full_name='caffe.NetStateRule.phase', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='min_level', full_name='caffe.NetStateRule.min_level', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='max_level', full_name='caffe.NetStateRule.max_level', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stage', full_name='caffe.NetStateRule.stage', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='not_stage', full_name='caffe.NetStateRule.not_stage', index=4, - number=5, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=2486, - serialized_end=2601, -) - - -_PARAMSPEC = _descriptor.Descriptor( - name='ParamSpec', - full_name='caffe.ParamSpec', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='caffe.ParamSpec.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='share_mode', full_name='caffe.ParamSpec.share_mode', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='lr_mult', full_name='caffe.ParamSpec.lr_mult', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='decay_mult', full_name='caffe.ParamSpec.decay_mult', index=3, - number=4, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _PARAMSPEC_DIMCHECKMODE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=2604, - serialized_end=2767, -) - - -_LAYERPARAMETER = _descriptor.Descriptor( - name='LayerParameter', - full_name='caffe.LayerParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='caffe.LayerParameter.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='type', full_name='caffe.LayerParameter.type', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bottom', full_name='caffe.LayerParameter.bottom', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='top', full_name='caffe.LayerParameter.top', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='phase', full_name='caffe.LayerParameter.phase', index=4, - number=10, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='loss_weight', full_name='caffe.LayerParameter.loss_weight', index=5, - number=5, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='param', full_name='caffe.LayerParameter.param', index=6, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='blobs', full_name='caffe.LayerParameter.blobs', index=7, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='propagate_down', full_name='caffe.LayerParameter.propagate_down', index=8, - number=11, type=8, cpp_type=7, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='include', full_name='caffe.LayerParameter.include', index=9, - number=8, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='exclude', full_name='caffe.LayerParameter.exclude', index=10, - number=9, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='transform_param', full_name='caffe.LayerParameter.transform_param', index=11, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='loss_param', full_name='caffe.LayerParameter.loss_param', index=12, - number=101, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='accuracy_param', full_name='caffe.LayerParameter.accuracy_param', index=13, - number=102, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='argmax_param', full_name='caffe.LayerParameter.argmax_param', index=14, - number=103, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='batch_norm_param', full_name='caffe.LayerParameter.batch_norm_param', index=15, - number=139, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_param', full_name='caffe.LayerParameter.bias_param', index=16, - number=141, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='concat_param', full_name='caffe.LayerParameter.concat_param', index=17, - number=104, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='contrastive_loss_param', full_name='caffe.LayerParameter.contrastive_loss_param', index=18, - number=105, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='convolution_param', full_name='caffe.LayerParameter.convolution_param', index=19, - number=106, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='crop_param', full_name='caffe.LayerParameter.crop_param', index=20, - number=144, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='data_param', full_name='caffe.LayerParameter.data_param', index=21, - number=107, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dropout_param', full_name='caffe.LayerParameter.dropout_param', index=22, - number=108, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dummy_data_param', full_name='caffe.LayerParameter.dummy_data_param', index=23, - number=109, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='eltwise_param', full_name='caffe.LayerParameter.eltwise_param', index=24, - number=110, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='elu_param', full_name='caffe.LayerParameter.elu_param', index=25, - number=140, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='embed_param', full_name='caffe.LayerParameter.embed_param', index=26, - number=137, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='exp_param', full_name='caffe.LayerParameter.exp_param', index=27, - number=111, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='flatten_param', full_name='caffe.LayerParameter.flatten_param', index=28, - number=135, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hdf5_data_param', full_name='caffe.LayerParameter.hdf5_data_param', index=29, - number=112, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hdf5_output_param', full_name='caffe.LayerParameter.hdf5_output_param', index=30, - number=113, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hinge_loss_param', full_name='caffe.LayerParameter.hinge_loss_param', index=31, - number=114, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='image_data_param', full_name='caffe.LayerParameter.image_data_param', index=32, - number=115, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='infogain_loss_param', full_name='caffe.LayerParameter.infogain_loss_param', index=33, - number=116, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='inner_product_param', full_name='caffe.LayerParameter.inner_product_param', index=34, - number=117, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='input_param', full_name='caffe.LayerParameter.input_param', index=35, - number=143, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='log_param', full_name='caffe.LayerParameter.log_param', index=36, - number=134, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='lrn_param', full_name='caffe.LayerParameter.lrn_param', index=37, - number=118, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='memory_data_param', full_name='caffe.LayerParameter.memory_data_param', index=38, - number=119, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mvn_param', full_name='caffe.LayerParameter.mvn_param', index=39, - number=120, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pooling_param', full_name='caffe.LayerParameter.pooling_param', index=40, - number=121, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='power_param', full_name='caffe.LayerParameter.power_param', index=41, - number=122, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='prelu_param', full_name='caffe.LayerParameter.prelu_param', index=42, - number=131, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='python_param', full_name='caffe.LayerParameter.python_param', index=43, - number=130, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='reduction_param', full_name='caffe.LayerParameter.reduction_param', index=44, - number=136, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='relu_param', full_name='caffe.LayerParameter.relu_param', index=45, - number=123, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='reshape_param', full_name='caffe.LayerParameter.reshape_param', index=46, - number=133, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale_param', full_name='caffe.LayerParameter.scale_param', index=47, - number=142, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sigmoid_param', full_name='caffe.LayerParameter.sigmoid_param', index=48, - number=124, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='softmax_param', full_name='caffe.LayerParameter.softmax_param', index=49, - number=125, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='spp_param', full_name='caffe.LayerParameter.spp_param', index=50, - number=132, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='slice_param', full_name='caffe.LayerParameter.slice_param', index=51, - number=126, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='tanh_param', full_name='caffe.LayerParameter.tanh_param', index=52, - number=127, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='threshold_param', full_name='caffe.LayerParameter.threshold_param', index=53, - number=128, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='tile_param', full_name='caffe.LayerParameter.tile_param', index=54, - number=138, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='window_data_param', full_name='caffe.LayerParameter.window_data_param', index=55, - number=129, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=2770, - serialized_end=5226, -) - - -_TRANSFORMATIONPARAMETER = _descriptor.Descriptor( - name='TransformationParameter', - full_name='caffe.TransformationParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.TransformationParameter.scale', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mirror', full_name='caffe.TransformationParameter.mirror', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='crop_size', full_name='caffe.TransformationParameter.crop_size', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mean_file', full_name='caffe.TransformationParameter.mean_file', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mean_value', full_name='caffe.TransformationParameter.mean_value', index=4, - number=5, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='force_color', full_name='caffe.TransformationParameter.force_color', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='force_gray', full_name='caffe.TransformationParameter.force_gray', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5229, - serialized_end=5411, -) - - -_LOSSPARAMETER = _descriptor.Descriptor( - name='LossParameter', - full_name='caffe.LossParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='ignore_label', full_name='caffe.LossParameter.ignore_label', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='normalization', full_name='caffe.LossParameter.normalization', index=1, - number=3, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='normalize', full_name='caffe.LossParameter.normalize', index=2, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _LOSSPARAMETER_NORMALIZATIONMODE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5414, - serialized_end=5608, -) - - -_ACCURACYPARAMETER = _descriptor.Descriptor( - name='AccuracyParameter', - full_name='caffe.AccuracyParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='top_k', full_name='caffe.AccuracyParameter.top_k', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.AccuracyParameter.axis', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='ignore_label', full_name='caffe.AccuracyParameter.ignore_label', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5610, - serialized_end=5686, -) - - -_ARGMAXPARAMETER = _descriptor.Descriptor( - name='ArgMaxParameter', - full_name='caffe.ArgMaxParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='out_max_val', full_name='caffe.ArgMaxParameter.out_max_val', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='top_k', full_name='caffe.ArgMaxParameter.top_k', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.ArgMaxParameter.axis', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5688, - serialized_end=5765, -) - - -_CONCATPARAMETER = _descriptor.Descriptor( - name='ConcatParameter', - full_name='caffe.ConcatParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.ConcatParameter.axis', index=0, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='concat_dim', full_name='caffe.ConcatParameter.concat_dim', index=1, - number=1, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5767, - serialized_end=5824, -) - - -_BATCHNORMPARAMETER = _descriptor.Descriptor( - name='BatchNormParameter', - full_name='caffe.BatchNormParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='use_global_stats', full_name='caffe.BatchNormParameter.use_global_stats', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='moving_average_fraction', full_name='caffe.BatchNormParameter.moving_average_fraction', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.999, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='eps', full_name='caffe.BatchNormParameter.eps', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1e-05, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5826, - serialized_end=5932, -) - - -_BIASPARAMETER = _descriptor.Descriptor( - name='BiasParameter', - full_name='caffe.BiasParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.BiasParameter.axis', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num_axes', full_name='caffe.BiasParameter.num_axes', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='filler', full_name='caffe.BiasParameter.filler', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=5934, - serialized_end=6027, -) - - -_CONTRASTIVELOSSPARAMETER = _descriptor.Descriptor( - name='ContrastiveLossParameter', - full_name='caffe.ContrastiveLossParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='margin', full_name='caffe.ContrastiveLossParameter.margin', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='legacy_version', full_name='caffe.ContrastiveLossParameter.legacy_version', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=6029, - serialized_end=6105, -) - - -_CONVOLUTIONPARAMETER = _descriptor.Descriptor( - name='ConvolutionParameter', - full_name='caffe.ConvolutionParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='num_output', full_name='caffe.ConvolutionParameter.num_output', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_term', full_name='caffe.ConvolutionParameter.bias_term', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad', full_name='caffe.ConvolutionParameter.pad', index=2, - number=3, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernel_size', full_name='caffe.ConvolutionParameter.kernel_size', index=3, - number=4, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride', full_name='caffe.ConvolutionParameter.stride', index=4, - number=6, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dilation', full_name='caffe.ConvolutionParameter.dilation', index=5, - number=18, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad_h', full_name='caffe.ConvolutionParameter.pad_h', index=6, - number=9, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad_w', full_name='caffe.ConvolutionParameter.pad_w', index=7, - number=10, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernel_h', full_name='caffe.ConvolutionParameter.kernel_h', index=8, - number=11, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernel_w', full_name='caffe.ConvolutionParameter.kernel_w', index=9, - number=12, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride_h', full_name='caffe.ConvolutionParameter.stride_h', index=10, - number=13, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride_w', full_name='caffe.ConvolutionParameter.stride_w', index=11, - number=14, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='group', full_name='caffe.ConvolutionParameter.group', index=12, - number=5, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_filler', full_name='caffe.ConvolutionParameter.weight_filler', index=13, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_filler', full_name='caffe.ConvolutionParameter.bias_filler', index=14, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.ConvolutionParameter.engine', index=15, - number=15, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.ConvolutionParameter.axis', index=16, - number=16, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='force_nd_im2col', full_name='caffe.ConvolutionParameter.force_nd_im2col', index=17, - number=17, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _CONVOLUTIONPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=6108, - serialized_end=6616, -) - - -_CROPPARAMETER = _descriptor.Descriptor( - name='CropParameter', - full_name='caffe.CropParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.CropParameter.axis', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=2, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='offset', full_name='caffe.CropParameter.offset', index=1, - number=2, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=6618, - serialized_end=6666, -) - - -_DATAPARAMETER = _descriptor.Descriptor( - name='DataParameter', - full_name='caffe.DataParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='caffe.DataParameter.source', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='batch_size', full_name='caffe.DataParameter.batch_size', index=1, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='rand_skip', full_name='caffe.DataParameter.rand_skip', index=2, - number=7, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='backend', full_name='caffe.DataParameter.backend', index=3, - number=8, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.DataParameter.scale', index=4, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mean_file', full_name='caffe.DataParameter.mean_file', index=5, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='crop_size', full_name='caffe.DataParameter.crop_size', index=6, - number=5, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mirror', full_name='caffe.DataParameter.mirror', index=7, - number=6, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='force_encoded_color', full_name='caffe.DataParameter.force_encoded_color', index=8, - number=9, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='prefetch', full_name='caffe.DataParameter.prefetch', index=9, - number=10, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=4, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _DATAPARAMETER_DB, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=6669, - serialized_end=6961, -) - - -_DROPOUTPARAMETER = _descriptor.Descriptor( - name='DropoutParameter', - full_name='caffe.DropoutParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='dropout_ratio', full_name='caffe.DropoutParameter.dropout_ratio', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=6963, - serialized_end=7009, -) - - -_DUMMYDATAPARAMETER = _descriptor.Descriptor( - name='DummyDataParameter', - full_name='caffe.DummyDataParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='data_filler', full_name='caffe.DummyDataParameter.data_filler', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shape', full_name='caffe.DummyDataParameter.shape', index=1, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num', full_name='caffe.DummyDataParameter.num', index=2, - number=2, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='channels', full_name='caffe.DummyDataParameter.channels', index=3, - number=3, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='height', full_name='caffe.DummyDataParameter.height', index=4, - number=4, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='width', full_name='caffe.DummyDataParameter.width', index=5, - number=5, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7012, - serialized_end=7172, -) - - -_ELTWISEPARAMETER = _descriptor.Descriptor( - name='EltwiseParameter', - full_name='caffe.EltwiseParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='operation', full_name='caffe.EltwiseParameter.operation', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='coeff', full_name='caffe.EltwiseParameter.coeff', index=1, - number=2, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stable_prod_grad', full_name='caffe.EltwiseParameter.stable_prod_grad', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _ELTWISEPARAMETER_ELTWISEOP, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7175, - serialized_end=7340, -) - - -_ELUPARAMETER = _descriptor.Descriptor( - name='ELUParameter', - full_name='caffe.ELUParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='alpha', full_name='caffe.ELUParameter.alpha', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7342, - serialized_end=7374, -) - - -_EMBEDPARAMETER = _descriptor.Descriptor( - name='EmbedParameter', - full_name='caffe.EmbedParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='num_output', full_name='caffe.EmbedParameter.num_output', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='input_dim', full_name='caffe.EmbedParameter.input_dim', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_term', full_name='caffe.EmbedParameter.bias_term', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_filler', full_name='caffe.EmbedParameter.weight_filler', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_filler', full_name='caffe.EmbedParameter.bias_filler', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7377, - serialized_end=7549, -) - - -_EXPPARAMETER = _descriptor.Descriptor( - name='ExpParameter', - full_name='caffe.ExpParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='base', full_name='caffe.ExpParameter.base', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.ExpParameter.scale', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shift', full_name='caffe.ExpParameter.shift', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7551, - serialized_end=7619, -) - - -_FLATTENPARAMETER = _descriptor.Descriptor( - name='FlattenParameter', - full_name='caffe.FlattenParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.FlattenParameter.axis', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_axis', full_name='caffe.FlattenParameter.end_axis', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7621, - serialized_end=7678, -) - - -_HDF5DATAPARAMETER = _descriptor.Descriptor( - name='HDF5DataParameter', - full_name='caffe.HDF5DataParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='caffe.HDF5DataParameter.source', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='batch_size', full_name='caffe.HDF5DataParameter.batch_size', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shuffle', full_name='caffe.HDF5DataParameter.shuffle', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7680, - serialized_end=7759, -) - - -_HDF5OUTPUTPARAMETER = _descriptor.Descriptor( - name='HDF5OutputParameter', - full_name='caffe.HDF5OutputParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='file_name', full_name='caffe.HDF5OutputParameter.file_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7761, - serialized_end=7801, -) - - -_HINGELOSSPARAMETER = _descriptor.Descriptor( - name='HingeLossParameter', - full_name='caffe.HingeLossParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='norm', full_name='caffe.HingeLossParameter.norm', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _HINGELOSSPARAMETER_NORM, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7803, - serialized_end=7897, -) - - -_IMAGEDATAPARAMETER = _descriptor.Descriptor( - name='ImageDataParameter', - full_name='caffe.ImageDataParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='caffe.ImageDataParameter.source', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='batch_size', full_name='caffe.ImageDataParameter.batch_size', index=1, - number=4, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='rand_skip', full_name='caffe.ImageDataParameter.rand_skip', index=2, - number=7, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shuffle', full_name='caffe.ImageDataParameter.shuffle', index=3, - number=8, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_height', full_name='caffe.ImageDataParameter.new_height', index=4, - number=9, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_width', full_name='caffe.ImageDataParameter.new_width', index=5, - number=10, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='is_color', full_name='caffe.ImageDataParameter.is_color', index=6, - number=11, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.ImageDataParameter.scale', index=7, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mean_file', full_name='caffe.ImageDataParameter.mean_file', index=8, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='crop_size', full_name='caffe.ImageDataParameter.crop_size', index=9, - number=5, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mirror', full_name='caffe.ImageDataParameter.mirror', index=10, - number=6, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='root_folder', full_name='caffe.ImageDataParameter.root_folder', index=11, - number=12, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=7900, - serialized_end=8179, -) - - -_INFOGAINLOSSPARAMETER = _descriptor.Descriptor( - name='InfogainLossParameter', - full_name='caffe.InfogainLossParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='caffe.InfogainLossParameter.source', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8181, - serialized_end=8220, -) - - -_INNERPRODUCTPARAMETER = _descriptor.Descriptor( - name='InnerProductParameter', - full_name='caffe.InnerProductParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='num_output', full_name='caffe.InnerProductParameter.num_output', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_term', full_name='caffe.InnerProductParameter.bias_term', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_filler', full_name='caffe.InnerProductParameter.weight_filler', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_filler', full_name='caffe.InnerProductParameter.bias_filler', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.InnerProductParameter.axis', index=4, - number=5, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='transpose', full_name='caffe.InnerProductParameter.transpose', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8223, - serialized_end=8426, -) - - -_INPUTPARAMETER = _descriptor.Descriptor( - name='InputParameter', - full_name='caffe.InputParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='shape', full_name='caffe.InputParameter.shape', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8428, - serialized_end=8477, -) - - -_LOGPARAMETER = _descriptor.Descriptor( - name='LogParameter', - full_name='caffe.LogParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='base', full_name='caffe.LogParameter.base', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.LogParameter.scale', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shift', full_name='caffe.LogParameter.shift', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8479, - serialized_end=8547, -) - - -_LRNPARAMETER = _descriptor.Descriptor( - name='LRNParameter', - full_name='caffe.LRNParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='local_size', full_name='caffe.LRNParameter.local_size', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='alpha', full_name='caffe.LRNParameter.alpha', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='beta', full_name='caffe.LRNParameter.beta', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.75, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='norm_region', full_name='caffe.LRNParameter.norm_region', index=3, - number=4, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='k', full_name='caffe.LRNParameter.k', index=4, - number=5, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.LRNParameter.engine', index=5, - number=6, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _LRNPARAMETER_NORMREGION, - _LRNPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8550, - serialized_end=8862, -) - - -_MEMORYDATAPARAMETER = _descriptor.Descriptor( - name='MemoryDataParameter', - full_name='caffe.MemoryDataParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='batch_size', full_name='caffe.MemoryDataParameter.batch_size', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='channels', full_name='caffe.MemoryDataParameter.channels', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='height', full_name='caffe.MemoryDataParameter.height', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='width', full_name='caffe.MemoryDataParameter.width', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8864, - serialized_end=8954, -) - - -_MVNPARAMETER = _descriptor.Descriptor( - name='MVNParameter', - full_name='caffe.MVNParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='normalize_variance', full_name='caffe.MVNParameter.normalize_variance', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='across_channels', full_name='caffe.MVNParameter.across_channels', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='eps', full_name='caffe.MVNParameter.eps', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1e-09, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=8956, - serialized_end=9056, -) - - -_POOLINGPARAMETER = _descriptor.Descriptor( - name='PoolingParameter', - full_name='caffe.PoolingParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='pool', full_name='caffe.PoolingParameter.pool', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad', full_name='caffe.PoolingParameter.pad', index=1, - number=4, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad_h', full_name='caffe.PoolingParameter.pad_h', index=2, - number=9, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad_w', full_name='caffe.PoolingParameter.pad_w', index=3, - number=10, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernel_size', full_name='caffe.PoolingParameter.kernel_size', index=4, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernel_h', full_name='caffe.PoolingParameter.kernel_h', index=5, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernel_w', full_name='caffe.PoolingParameter.kernel_w', index=6, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride', full_name='caffe.PoolingParameter.stride', index=7, - number=3, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride_h', full_name='caffe.PoolingParameter.stride_h', index=8, - number=7, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride_w', full_name='caffe.PoolingParameter.stride_w', index=9, - number=8, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.PoolingParameter.engine', index=10, - number=11, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='global_pooling', full_name='caffe.PoolingParameter.global_pooling', index=11, - number=12, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _POOLINGPARAMETER_POOLMETHOD, - _POOLINGPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=9059, - serialized_end=9477, -) - - -_POWERPARAMETER = _descriptor.Descriptor( - name='PowerParameter', - full_name='caffe.PowerParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='power', full_name='caffe.PowerParameter.power', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.PowerParameter.scale', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shift', full_name='caffe.PowerParameter.shift', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=9479, - serialized_end=9549, -) - - -_PYTHONPARAMETER = _descriptor.Descriptor( - name='PythonParameter', - full_name='caffe.PythonParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='module', full_name='caffe.PythonParameter.module', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='layer', full_name='caffe.PythonParameter.layer', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='param_str', full_name='caffe.PythonParameter.param_str', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='share_in_parallel', full_name='caffe.PythonParameter.share_in_parallel', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=9551, - serialized_end=9654, -) - - -_REDUCTIONPARAMETER = _descriptor.Descriptor( - name='ReductionParameter', - full_name='caffe.ReductionParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='operation', full_name='caffe.ReductionParameter.operation', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.ReductionParameter.axis', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='coeff', full_name='caffe.ReductionParameter.coeff', index=2, - number=3, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _REDUCTIONPARAMETER_REDUCTIONOP, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=9657, - serialized_end=9830, -) - - -_RELUPARAMETER = _descriptor.Descriptor( - name='ReLUParameter', - full_name='caffe.ReLUParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='negative_slope', full_name='caffe.ReLUParameter.negative_slope', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.ReLUParameter.engine', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _RELUPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=9833, - serialized_end=9974, -) - - -_RESHAPEPARAMETER = _descriptor.Descriptor( - name='ReshapeParameter', - full_name='caffe.ReshapeParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='shape', full_name='caffe.ReshapeParameter.shape', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.ReshapeParameter.axis', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num_axes', full_name='caffe.ReshapeParameter.num_axes', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=-1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=9976, - serialized_end=10066, -) - - -_SCALEPARAMETER = _descriptor.Descriptor( - name='ScaleParameter', - full_name='caffe.ScaleParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.ScaleParameter.axis', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num_axes', full_name='caffe.ScaleParameter.num_axes', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='filler', full_name='caffe.ScaleParameter.filler', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_term', full_name='caffe.ScaleParameter.bias_term', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_filler', full_name='caffe.ScaleParameter.bias_filler', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10069, - serialized_end=10234, -) - - -_SIGMOIDPARAMETER = _descriptor.Descriptor( - name='SigmoidParameter', - full_name='caffe.SigmoidParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.SigmoidParameter.engine', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SIGMOIDPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10236, - serialized_end=10356, -) - - -_SLICEPARAMETER = _descriptor.Descriptor( - name='SliceParameter', - full_name='caffe.SliceParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.SliceParameter.axis', index=0, - number=3, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='slice_point', full_name='caffe.SliceParameter.slice_point', index=1, - number=2, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='slice_dim', full_name='caffe.SliceParameter.slice_dim', index=2, - number=1, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10358, - serialized_end=10434, -) - - -_SOFTMAXPARAMETER = _descriptor.Descriptor( - name='SoftmaxParameter', - full_name='caffe.SoftmaxParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.SoftmaxParameter.engine', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.SoftmaxParameter.axis', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SOFTMAXPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10437, - serialized_end=10574, -) - - -_TANHPARAMETER = _descriptor.Descriptor( - name='TanHParameter', - full_name='caffe.TanHParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.TanHParameter.engine', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _TANHPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10576, - serialized_end=10690, -) - - -_TILEPARAMETER = _descriptor.Descriptor( - name='TileParameter', - full_name='caffe.TileParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='axis', full_name='caffe.TileParameter.axis', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='tiles', full_name='caffe.TileParameter.tiles', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10692, - serialized_end=10739, -) - - -_THRESHOLDPARAMETER = _descriptor.Descriptor( - name='ThresholdParameter', - full_name='caffe.ThresholdParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='threshold', full_name='caffe.ThresholdParameter.threshold', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10741, - serialized_end=10783, -) - - -_WINDOWDATAPARAMETER = _descriptor.Descriptor( - name='WindowDataParameter', - full_name='caffe.WindowDataParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='caffe.WindowDataParameter.source', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.WindowDataParameter.scale', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mean_file', full_name='caffe.WindowDataParameter.mean_file', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='batch_size', full_name='caffe.WindowDataParameter.batch_size', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='crop_size', full_name='caffe.WindowDataParameter.crop_size', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mirror', full_name='caffe.WindowDataParameter.mirror', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='fg_threshold', full_name='caffe.WindowDataParameter.fg_threshold', index=6, - number=7, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bg_threshold', full_name='caffe.WindowDataParameter.bg_threshold', index=7, - number=8, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='fg_fraction', full_name='caffe.WindowDataParameter.fg_fraction', index=8, - number=9, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.25, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='context_pad', full_name='caffe.WindowDataParameter.context_pad', index=9, - number=10, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='crop_mode', full_name='caffe.WindowDataParameter.crop_mode', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("warp", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cache_images', full_name='caffe.WindowDataParameter.cache_images', index=11, - number=12, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='root_folder', full_name='caffe.WindowDataParameter.root_folder', index=12, - number=13, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=10786, - serialized_end=11107, -) - - -_SPPPARAMETER = _descriptor.Descriptor( - name='SPPParameter', - full_name='caffe.SPPParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='pyramid_height', full_name='caffe.SPPParameter.pyramid_height', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pool', full_name='caffe.SPPParameter.pool', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='engine', full_name='caffe.SPPParameter.engine', index=2, - number=6, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SPPPARAMETER_POOLMETHOD, - _SPPPARAMETER_ENGINE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=11110, - serialized_end=11345, -) - - -_V1LAYERPARAMETER = _descriptor.Descriptor( - name='V1LayerParameter', - full_name='caffe.V1LayerParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='bottom', full_name='caffe.V1LayerParameter.bottom', index=0, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='top', full_name='caffe.V1LayerParameter.top', index=1, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='name', full_name='caffe.V1LayerParameter.name', index=2, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='include', full_name='caffe.V1LayerParameter.include', index=3, - number=32, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='exclude', full_name='caffe.V1LayerParameter.exclude', index=4, - number=33, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='type', full_name='caffe.V1LayerParameter.type', index=5, - number=5, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='blobs', full_name='caffe.V1LayerParameter.blobs', index=6, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='param', full_name='caffe.V1LayerParameter.param', index=7, - number=1001, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='blob_share_mode', full_name='caffe.V1LayerParameter.blob_share_mode', index=8, - number=1002, type=14, cpp_type=8, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='blobs_lr', full_name='caffe.V1LayerParameter.blobs_lr', index=9, - number=7, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_decay', full_name='caffe.V1LayerParameter.weight_decay', index=10, - number=8, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='loss_weight', full_name='caffe.V1LayerParameter.loss_weight', index=11, - number=35, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='accuracy_param', full_name='caffe.V1LayerParameter.accuracy_param', index=12, - number=27, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='argmax_param', full_name='caffe.V1LayerParameter.argmax_param', index=13, - number=23, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='concat_param', full_name='caffe.V1LayerParameter.concat_param', index=14, - number=9, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='contrastive_loss_param', full_name='caffe.V1LayerParameter.contrastive_loss_param', index=15, - number=40, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='convolution_param', full_name='caffe.V1LayerParameter.convolution_param', index=16, - number=10, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='data_param', full_name='caffe.V1LayerParameter.data_param', index=17, - number=11, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dropout_param', full_name='caffe.V1LayerParameter.dropout_param', index=18, - number=12, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dummy_data_param', full_name='caffe.V1LayerParameter.dummy_data_param', index=19, - number=26, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='eltwise_param', full_name='caffe.V1LayerParameter.eltwise_param', index=20, - number=24, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='exp_param', full_name='caffe.V1LayerParameter.exp_param', index=21, - number=41, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hdf5_data_param', full_name='caffe.V1LayerParameter.hdf5_data_param', index=22, - number=13, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hdf5_output_param', full_name='caffe.V1LayerParameter.hdf5_output_param', index=23, - number=14, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hinge_loss_param', full_name='caffe.V1LayerParameter.hinge_loss_param', index=24, - number=29, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='image_data_param', full_name='caffe.V1LayerParameter.image_data_param', index=25, - number=15, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='infogain_loss_param', full_name='caffe.V1LayerParameter.infogain_loss_param', index=26, - number=16, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='inner_product_param', full_name='caffe.V1LayerParameter.inner_product_param', index=27, - number=17, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='lrn_param', full_name='caffe.V1LayerParameter.lrn_param', index=28, - number=18, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='memory_data_param', full_name='caffe.V1LayerParameter.memory_data_param', index=29, - number=22, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mvn_param', full_name='caffe.V1LayerParameter.mvn_param', index=30, - number=34, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pooling_param', full_name='caffe.V1LayerParameter.pooling_param', index=31, - number=19, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='power_param', full_name='caffe.V1LayerParameter.power_param', index=32, - number=21, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='relu_param', full_name='caffe.V1LayerParameter.relu_param', index=33, - number=30, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sigmoid_param', full_name='caffe.V1LayerParameter.sigmoid_param', index=34, - number=38, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='softmax_param', full_name='caffe.V1LayerParameter.softmax_param', index=35, - number=39, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='slice_param', full_name='caffe.V1LayerParameter.slice_param', index=36, - number=31, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='tanh_param', full_name='caffe.V1LayerParameter.tanh_param', index=37, - number=37, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='threshold_param', full_name='caffe.V1LayerParameter.threshold_param', index=38, - number=25, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='window_data_param', full_name='caffe.V1LayerParameter.window_data_param', index=39, - number=20, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='transform_param', full_name='caffe.V1LayerParameter.transform_param', index=40, - number=36, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='loss_param', full_name='caffe.V1LayerParameter.loss_param', index=41, - number=42, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='layer', full_name='caffe.V1LayerParameter.layer', index=42, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _V1LAYERPARAMETER_LAYERTYPE, - _V1LAYERPARAMETER_DIMCHECKMODE, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=11348, - serialized_end=13876, -) - - -_V0LAYERPARAMETER = _descriptor.Descriptor( - name='V0LayerParameter', - full_name='caffe.V0LayerParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='caffe.V0LayerParameter.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='type', full_name='caffe.V0LayerParameter.type', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num_output', full_name='caffe.V0LayerParameter.num_output', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='biasterm', full_name='caffe.V0LayerParameter.biasterm', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_filler', full_name='caffe.V0LayerParameter.weight_filler', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bias_filler', full_name='caffe.V0LayerParameter.bias_filler', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pad', full_name='caffe.V0LayerParameter.pad', index=6, - number=7, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='kernelsize', full_name='caffe.V0LayerParameter.kernelsize', index=7, - number=8, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='group', full_name='caffe.V0LayerParameter.group', index=8, - number=9, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='stride', full_name='caffe.V0LayerParameter.stride', index=9, - number=10, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pool', full_name='caffe.V0LayerParameter.pool', index=10, - number=11, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dropout_ratio', full_name='caffe.V0LayerParameter.dropout_ratio', index=11, - number=12, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='local_size', full_name='caffe.V0LayerParameter.local_size', index=12, - number=13, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='alpha', full_name='caffe.V0LayerParameter.alpha', index=13, - number=14, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='beta', full_name='caffe.V0LayerParameter.beta', index=14, - number=15, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.75, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='k', full_name='caffe.V0LayerParameter.k', index=15, - number=22, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='source', full_name='caffe.V0LayerParameter.source', index=16, - number=16, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scale', full_name='caffe.V0LayerParameter.scale', index=17, - number=17, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='meanfile', full_name='caffe.V0LayerParameter.meanfile', index=18, - number=18, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='batchsize', full_name='caffe.V0LayerParameter.batchsize', index=19, - number=19, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cropsize', full_name='caffe.V0LayerParameter.cropsize', index=20, - number=20, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mirror', full_name='caffe.V0LayerParameter.mirror', index=21, - number=21, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='blobs', full_name='caffe.V0LayerParameter.blobs', index=22, - number=50, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='blobs_lr', full_name='caffe.V0LayerParameter.blobs_lr', index=23, - number=51, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weight_decay', full_name='caffe.V0LayerParameter.weight_decay', index=24, - number=52, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='rand_skip', full_name='caffe.V0LayerParameter.rand_skip', index=25, - number=53, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='det_fg_threshold', full_name='caffe.V0LayerParameter.det_fg_threshold', index=26, - number=54, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='det_bg_threshold', full_name='caffe.V0LayerParameter.det_bg_threshold', index=27, - number=55, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.5, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='det_fg_fraction', full_name='caffe.V0LayerParameter.det_fg_fraction', index=28, - number=56, type=2, cpp_type=6, label=1, - has_default_value=True, default_value=0.25, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='det_context_pad', full_name='caffe.V0LayerParameter.det_context_pad', index=29, - number=58, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='det_crop_mode', full_name='caffe.V0LayerParameter.det_crop_mode', index=30, - number=59, type=9, cpp_type=9, label=1, - has_default_value=True, default_value=unicode("warp", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_num', full_name='caffe.V0LayerParameter.new_num', index=31, - number=60, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_channels', full_name='caffe.V0LayerParameter.new_channels', index=32, - number=61, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_height', full_name='caffe.V0LayerParameter.new_height', index=33, - number=62, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_width', full_name='caffe.V0LayerParameter.new_width', index=34, - number=63, type=5, cpp_type=1, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='shuffle_images', full_name='caffe.V0LayerParameter.shuffle_images', index=35, - number=64, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='concat_dim', full_name='caffe.V0LayerParameter.concat_dim', index=36, - number=65, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='hdf5_output_param', full_name='caffe.V0LayerParameter.hdf5_output_param', index=37, - number=1001, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _V0LAYERPARAMETER_POOLMETHOD, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=13879, - serialized_end=14900, -) - - -_PRELUPARAMETER = _descriptor.Descriptor( - name='PReLUParameter', - full_name='caffe.PReLUParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='filler', full_name='caffe.PReLUParameter.filler', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='channel_shared', full_name='caffe.PReLUParameter.channel_shared', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=14902, - serialized_end=14989, -) - -_BLOBPROTO.fields_by_name['shape'].message_type = _BLOBSHAPE -_BLOBPROTOVECTOR.fields_by_name['blobs'].message_type = _BLOBPROTO -_FILLERPARAMETER.fields_by_name['variance_norm'].enum_type = _FILLERPARAMETER_VARIANCENORM -_FILLERPARAMETER_VARIANCENORM.containing_type = _FILLERPARAMETER; -_NETPARAMETER.fields_by_name['input_shape'].message_type = _BLOBSHAPE -_NETPARAMETER.fields_by_name['state'].message_type = _NETSTATE -_NETPARAMETER.fields_by_name['layer'].message_type = _LAYERPARAMETER -_NETPARAMETER.fields_by_name['layers'].message_type = _V1LAYERPARAMETER -_SOLVERPARAMETER.fields_by_name['net_param'].message_type = _NETPARAMETER -_SOLVERPARAMETER.fields_by_name['train_net_param'].message_type = _NETPARAMETER -_SOLVERPARAMETER.fields_by_name['test_net_param'].message_type = _NETPARAMETER -_SOLVERPARAMETER.fields_by_name['train_state'].message_type = _NETSTATE -_SOLVERPARAMETER.fields_by_name['test_state'].message_type = _NETSTATE -_SOLVERPARAMETER.fields_by_name['snapshot_format'].enum_type = _SOLVERPARAMETER_SNAPSHOTFORMAT -_SOLVERPARAMETER.fields_by_name['solver_mode'].enum_type = _SOLVERPARAMETER_SOLVERMODE -_SOLVERPARAMETER.fields_by_name['solver_type'].enum_type = _SOLVERPARAMETER_SOLVERTYPE -_SOLVERPARAMETER_SNAPSHOTFORMAT.containing_type = _SOLVERPARAMETER; -_SOLVERPARAMETER_SOLVERMODE.containing_type = _SOLVERPARAMETER; -_SOLVERPARAMETER_SOLVERTYPE.containing_type = _SOLVERPARAMETER; -_SOLVERSTATE.fields_by_name['history'].message_type = _BLOBPROTO -_NETSTATE.fields_by_name['phase'].enum_type = _PHASE -_NETSTATERULE.fields_by_name['phase'].enum_type = _PHASE -_PARAMSPEC.fields_by_name['share_mode'].enum_type = _PARAMSPEC_DIMCHECKMODE -_PARAMSPEC_DIMCHECKMODE.containing_type = _PARAMSPEC; -_LAYERPARAMETER.fields_by_name['phase'].enum_type = _PHASE -_LAYERPARAMETER.fields_by_name['param'].message_type = _PARAMSPEC -_LAYERPARAMETER.fields_by_name['blobs'].message_type = _BLOBPROTO -_LAYERPARAMETER.fields_by_name['include'].message_type = _NETSTATERULE -_LAYERPARAMETER.fields_by_name['exclude'].message_type = _NETSTATERULE -_LAYERPARAMETER.fields_by_name['transform_param'].message_type = _TRANSFORMATIONPARAMETER -_LAYERPARAMETER.fields_by_name['loss_param'].message_type = _LOSSPARAMETER -_LAYERPARAMETER.fields_by_name['accuracy_param'].message_type = _ACCURACYPARAMETER -_LAYERPARAMETER.fields_by_name['argmax_param'].message_type = _ARGMAXPARAMETER -_LAYERPARAMETER.fields_by_name['batch_norm_param'].message_type = _BATCHNORMPARAMETER -_LAYERPARAMETER.fields_by_name['bias_param'].message_type = _BIASPARAMETER -_LAYERPARAMETER.fields_by_name['concat_param'].message_type = _CONCATPARAMETER -_LAYERPARAMETER.fields_by_name['contrastive_loss_param'].message_type = _CONTRASTIVELOSSPARAMETER -_LAYERPARAMETER.fields_by_name['convolution_param'].message_type = _CONVOLUTIONPARAMETER -_LAYERPARAMETER.fields_by_name['crop_param'].message_type = _CROPPARAMETER -_LAYERPARAMETER.fields_by_name['data_param'].message_type = _DATAPARAMETER -_LAYERPARAMETER.fields_by_name['dropout_param'].message_type = _DROPOUTPARAMETER -_LAYERPARAMETER.fields_by_name['dummy_data_param'].message_type = _DUMMYDATAPARAMETER -_LAYERPARAMETER.fields_by_name['eltwise_param'].message_type = _ELTWISEPARAMETER -_LAYERPARAMETER.fields_by_name['elu_param'].message_type = _ELUPARAMETER -_LAYERPARAMETER.fields_by_name['embed_param'].message_type = _EMBEDPARAMETER -_LAYERPARAMETER.fields_by_name['exp_param'].message_type = _EXPPARAMETER -_LAYERPARAMETER.fields_by_name['flatten_param'].message_type = _FLATTENPARAMETER -_LAYERPARAMETER.fields_by_name['hdf5_data_param'].message_type = _HDF5DATAPARAMETER -_LAYERPARAMETER.fields_by_name['hdf5_output_param'].message_type = _HDF5OUTPUTPARAMETER -_LAYERPARAMETER.fields_by_name['hinge_loss_param'].message_type = _HINGELOSSPARAMETER -_LAYERPARAMETER.fields_by_name['image_data_param'].message_type = _IMAGEDATAPARAMETER -_LAYERPARAMETER.fields_by_name['infogain_loss_param'].message_type = _INFOGAINLOSSPARAMETER -_LAYERPARAMETER.fields_by_name['inner_product_param'].message_type = _INNERPRODUCTPARAMETER -_LAYERPARAMETER.fields_by_name['input_param'].message_type = _INPUTPARAMETER -_LAYERPARAMETER.fields_by_name['log_param'].message_type = _LOGPARAMETER -_LAYERPARAMETER.fields_by_name['lrn_param'].message_type = _LRNPARAMETER -_LAYERPARAMETER.fields_by_name['memory_data_param'].message_type = _MEMORYDATAPARAMETER -_LAYERPARAMETER.fields_by_name['mvn_param'].message_type = _MVNPARAMETER -_LAYERPARAMETER.fields_by_name['pooling_param'].message_type = _POOLINGPARAMETER -_LAYERPARAMETER.fields_by_name['power_param'].message_type = _POWERPARAMETER -_LAYERPARAMETER.fields_by_name['prelu_param'].message_type = _PRELUPARAMETER -_LAYERPARAMETER.fields_by_name['python_param'].message_type = _PYTHONPARAMETER -_LAYERPARAMETER.fields_by_name['reduction_param'].message_type = _REDUCTIONPARAMETER -_LAYERPARAMETER.fields_by_name['relu_param'].message_type = _RELUPARAMETER -_LAYERPARAMETER.fields_by_name['reshape_param'].message_type = _RESHAPEPARAMETER -_LAYERPARAMETER.fields_by_name['scale_param'].message_type = _SCALEPARAMETER -_LAYERPARAMETER.fields_by_name['sigmoid_param'].message_type = _SIGMOIDPARAMETER -_LAYERPARAMETER.fields_by_name['softmax_param'].message_type = _SOFTMAXPARAMETER -_LAYERPARAMETER.fields_by_name['spp_param'].message_type = _SPPPARAMETER -_LAYERPARAMETER.fields_by_name['slice_param'].message_type = _SLICEPARAMETER -_LAYERPARAMETER.fields_by_name['tanh_param'].message_type = _TANHPARAMETER -_LAYERPARAMETER.fields_by_name['threshold_param'].message_type = _THRESHOLDPARAMETER -_LAYERPARAMETER.fields_by_name['tile_param'].message_type = _TILEPARAMETER -_LAYERPARAMETER.fields_by_name['window_data_param'].message_type = _WINDOWDATAPARAMETER -_LOSSPARAMETER.fields_by_name['normalization'].enum_type = _LOSSPARAMETER_NORMALIZATIONMODE -_LOSSPARAMETER_NORMALIZATIONMODE.containing_type = _LOSSPARAMETER; -_BIASPARAMETER.fields_by_name['filler'].message_type = _FILLERPARAMETER -_CONVOLUTIONPARAMETER.fields_by_name['weight_filler'].message_type = _FILLERPARAMETER -_CONVOLUTIONPARAMETER.fields_by_name['bias_filler'].message_type = _FILLERPARAMETER -_CONVOLUTIONPARAMETER.fields_by_name['engine'].enum_type = _CONVOLUTIONPARAMETER_ENGINE -_CONVOLUTIONPARAMETER_ENGINE.containing_type = _CONVOLUTIONPARAMETER; -_DATAPARAMETER.fields_by_name['backend'].enum_type = _DATAPARAMETER_DB -_DATAPARAMETER_DB.containing_type = _DATAPARAMETER; -_DUMMYDATAPARAMETER.fields_by_name['data_filler'].message_type = _FILLERPARAMETER -_DUMMYDATAPARAMETER.fields_by_name['shape'].message_type = _BLOBSHAPE -_ELTWISEPARAMETER.fields_by_name['operation'].enum_type = _ELTWISEPARAMETER_ELTWISEOP -_ELTWISEPARAMETER_ELTWISEOP.containing_type = _ELTWISEPARAMETER; -_EMBEDPARAMETER.fields_by_name['weight_filler'].message_type = _FILLERPARAMETER -_EMBEDPARAMETER.fields_by_name['bias_filler'].message_type = _FILLERPARAMETER -_HINGELOSSPARAMETER.fields_by_name['norm'].enum_type = _HINGELOSSPARAMETER_NORM -_HINGELOSSPARAMETER_NORM.containing_type = _HINGELOSSPARAMETER; -_INNERPRODUCTPARAMETER.fields_by_name['weight_filler'].message_type = _FILLERPARAMETER -_INNERPRODUCTPARAMETER.fields_by_name['bias_filler'].message_type = _FILLERPARAMETER -_INPUTPARAMETER.fields_by_name['shape'].message_type = _BLOBSHAPE -_LRNPARAMETER.fields_by_name['norm_region'].enum_type = _LRNPARAMETER_NORMREGION -_LRNPARAMETER.fields_by_name['engine'].enum_type = _LRNPARAMETER_ENGINE -_LRNPARAMETER_NORMREGION.containing_type = _LRNPARAMETER; -_LRNPARAMETER_ENGINE.containing_type = _LRNPARAMETER; -_POOLINGPARAMETER.fields_by_name['pool'].enum_type = _POOLINGPARAMETER_POOLMETHOD -_POOLINGPARAMETER.fields_by_name['engine'].enum_type = _POOLINGPARAMETER_ENGINE -_POOLINGPARAMETER_POOLMETHOD.containing_type = _POOLINGPARAMETER; -_POOLINGPARAMETER_ENGINE.containing_type = _POOLINGPARAMETER; -_REDUCTIONPARAMETER.fields_by_name['operation'].enum_type = _REDUCTIONPARAMETER_REDUCTIONOP -_REDUCTIONPARAMETER_REDUCTIONOP.containing_type = _REDUCTIONPARAMETER; -_RELUPARAMETER.fields_by_name['engine'].enum_type = _RELUPARAMETER_ENGINE -_RELUPARAMETER_ENGINE.containing_type = _RELUPARAMETER; -_RESHAPEPARAMETER.fields_by_name['shape'].message_type = _BLOBSHAPE -_SCALEPARAMETER.fields_by_name['filler'].message_type = _FILLERPARAMETER -_SCALEPARAMETER.fields_by_name['bias_filler'].message_type = _FILLERPARAMETER -_SIGMOIDPARAMETER.fields_by_name['engine'].enum_type = _SIGMOIDPARAMETER_ENGINE -_SIGMOIDPARAMETER_ENGINE.containing_type = _SIGMOIDPARAMETER; -_SOFTMAXPARAMETER.fields_by_name['engine'].enum_type = _SOFTMAXPARAMETER_ENGINE -_SOFTMAXPARAMETER_ENGINE.containing_type = _SOFTMAXPARAMETER; -_TANHPARAMETER.fields_by_name['engine'].enum_type = _TANHPARAMETER_ENGINE -_TANHPARAMETER_ENGINE.containing_type = _TANHPARAMETER; -_SPPPARAMETER.fields_by_name['pool'].enum_type = _SPPPARAMETER_POOLMETHOD -_SPPPARAMETER.fields_by_name['engine'].enum_type = _SPPPARAMETER_ENGINE -_SPPPARAMETER_POOLMETHOD.containing_type = _SPPPARAMETER; -_SPPPARAMETER_ENGINE.containing_type = _SPPPARAMETER; -_V1LAYERPARAMETER.fields_by_name['include'].message_type = _NETSTATERULE -_V1LAYERPARAMETER.fields_by_name['exclude'].message_type = _NETSTATERULE -_V1LAYERPARAMETER.fields_by_name['type'].enum_type = _V1LAYERPARAMETER_LAYERTYPE -_V1LAYERPARAMETER.fields_by_name['blobs'].message_type = _BLOBPROTO -_V1LAYERPARAMETER.fields_by_name['blob_share_mode'].enum_type = _V1LAYERPARAMETER_DIMCHECKMODE -_V1LAYERPARAMETER.fields_by_name['accuracy_param'].message_type = _ACCURACYPARAMETER -_V1LAYERPARAMETER.fields_by_name['argmax_param'].message_type = _ARGMAXPARAMETER -_V1LAYERPARAMETER.fields_by_name['concat_param'].message_type = _CONCATPARAMETER -_V1LAYERPARAMETER.fields_by_name['contrastive_loss_param'].message_type = _CONTRASTIVELOSSPARAMETER -_V1LAYERPARAMETER.fields_by_name['convolution_param'].message_type = _CONVOLUTIONPARAMETER -_V1LAYERPARAMETER.fields_by_name['data_param'].message_type = _DATAPARAMETER -_V1LAYERPARAMETER.fields_by_name['dropout_param'].message_type = _DROPOUTPARAMETER -_V1LAYERPARAMETER.fields_by_name['dummy_data_param'].message_type = _DUMMYDATAPARAMETER -_V1LAYERPARAMETER.fields_by_name['eltwise_param'].message_type = _ELTWISEPARAMETER -_V1LAYERPARAMETER.fields_by_name['exp_param'].message_type = _EXPPARAMETER -_V1LAYERPARAMETER.fields_by_name['hdf5_data_param'].message_type = _HDF5DATAPARAMETER -_V1LAYERPARAMETER.fields_by_name['hdf5_output_param'].message_type = _HDF5OUTPUTPARAMETER -_V1LAYERPARAMETER.fields_by_name['hinge_loss_param'].message_type = _HINGELOSSPARAMETER -_V1LAYERPARAMETER.fields_by_name['image_data_param'].message_type = _IMAGEDATAPARAMETER -_V1LAYERPARAMETER.fields_by_name['infogain_loss_param'].message_type = _INFOGAINLOSSPARAMETER -_V1LAYERPARAMETER.fields_by_name['inner_product_param'].message_type = _INNERPRODUCTPARAMETER -_V1LAYERPARAMETER.fields_by_name['lrn_param'].message_type = _LRNPARAMETER -_V1LAYERPARAMETER.fields_by_name['memory_data_param'].message_type = _MEMORYDATAPARAMETER -_V1LAYERPARAMETER.fields_by_name['mvn_param'].message_type = _MVNPARAMETER -_V1LAYERPARAMETER.fields_by_name['pooling_param'].message_type = _POOLINGPARAMETER -_V1LAYERPARAMETER.fields_by_name['power_param'].message_type = _POWERPARAMETER -_V1LAYERPARAMETER.fields_by_name['relu_param'].message_type = _RELUPARAMETER -_V1LAYERPARAMETER.fields_by_name['sigmoid_param'].message_type = _SIGMOIDPARAMETER -_V1LAYERPARAMETER.fields_by_name['softmax_param'].message_type = _SOFTMAXPARAMETER -_V1LAYERPARAMETER.fields_by_name['slice_param'].message_type = _SLICEPARAMETER -_V1LAYERPARAMETER.fields_by_name['tanh_param'].message_type = _TANHPARAMETER -_V1LAYERPARAMETER.fields_by_name['threshold_param'].message_type = _THRESHOLDPARAMETER -_V1LAYERPARAMETER.fields_by_name['window_data_param'].message_type = _WINDOWDATAPARAMETER -_V1LAYERPARAMETER.fields_by_name['transform_param'].message_type = _TRANSFORMATIONPARAMETER -_V1LAYERPARAMETER.fields_by_name['loss_param'].message_type = _LOSSPARAMETER -_V1LAYERPARAMETER.fields_by_name['layer'].message_type = _V0LAYERPARAMETER -_V1LAYERPARAMETER_LAYERTYPE.containing_type = _V1LAYERPARAMETER; -_V1LAYERPARAMETER_DIMCHECKMODE.containing_type = _V1LAYERPARAMETER; -_V0LAYERPARAMETER.fields_by_name['weight_filler'].message_type = _FILLERPARAMETER -_V0LAYERPARAMETER.fields_by_name['bias_filler'].message_type = _FILLERPARAMETER -_V0LAYERPARAMETER.fields_by_name['pool'].enum_type = _V0LAYERPARAMETER_POOLMETHOD -_V0LAYERPARAMETER.fields_by_name['blobs'].message_type = _BLOBPROTO -_V0LAYERPARAMETER.fields_by_name['hdf5_output_param'].message_type = _HDF5OUTPUTPARAMETER -_V0LAYERPARAMETER_POOLMETHOD.containing_type = _V0LAYERPARAMETER; -_PRELUPARAMETER.fields_by_name['filler'].message_type = _FILLERPARAMETER -DESCRIPTOR.message_types_by_name['BlobShape'] = _BLOBSHAPE -DESCRIPTOR.message_types_by_name['BlobProto'] = _BLOBPROTO -DESCRIPTOR.message_types_by_name['BlobProtoVector'] = _BLOBPROTOVECTOR -DESCRIPTOR.message_types_by_name['Datum'] = _DATUM -DESCRIPTOR.message_types_by_name['FillerParameter'] = _FILLERPARAMETER -DESCRIPTOR.message_types_by_name['NetParameter'] = _NETPARAMETER -DESCRIPTOR.message_types_by_name['SolverParameter'] = _SOLVERPARAMETER -DESCRIPTOR.message_types_by_name['SolverState'] = _SOLVERSTATE -DESCRIPTOR.message_types_by_name['NetState'] = _NETSTATE -DESCRIPTOR.message_types_by_name['NetStateRule'] = _NETSTATERULE -DESCRIPTOR.message_types_by_name['ParamSpec'] = _PARAMSPEC -DESCRIPTOR.message_types_by_name['LayerParameter'] = _LAYERPARAMETER -DESCRIPTOR.message_types_by_name['TransformationParameter'] = _TRANSFORMATIONPARAMETER -DESCRIPTOR.message_types_by_name['LossParameter'] = _LOSSPARAMETER -DESCRIPTOR.message_types_by_name['AccuracyParameter'] = _ACCURACYPARAMETER -DESCRIPTOR.message_types_by_name['ArgMaxParameter'] = _ARGMAXPARAMETER -DESCRIPTOR.message_types_by_name['ConcatParameter'] = _CONCATPARAMETER -DESCRIPTOR.message_types_by_name['BatchNormParameter'] = _BATCHNORMPARAMETER -DESCRIPTOR.message_types_by_name['BiasParameter'] = _BIASPARAMETER -DESCRIPTOR.message_types_by_name['ContrastiveLossParameter'] = _CONTRASTIVELOSSPARAMETER -DESCRIPTOR.message_types_by_name['ConvolutionParameter'] = _CONVOLUTIONPARAMETER -DESCRIPTOR.message_types_by_name['CropParameter'] = _CROPPARAMETER -DESCRIPTOR.message_types_by_name['DataParameter'] = _DATAPARAMETER -DESCRIPTOR.message_types_by_name['DropoutParameter'] = _DROPOUTPARAMETER -DESCRIPTOR.message_types_by_name['DummyDataParameter'] = _DUMMYDATAPARAMETER -DESCRIPTOR.message_types_by_name['EltwiseParameter'] = _ELTWISEPARAMETER -DESCRIPTOR.message_types_by_name['ELUParameter'] = _ELUPARAMETER -DESCRIPTOR.message_types_by_name['EmbedParameter'] = _EMBEDPARAMETER -DESCRIPTOR.message_types_by_name['ExpParameter'] = _EXPPARAMETER -DESCRIPTOR.message_types_by_name['FlattenParameter'] = _FLATTENPARAMETER -DESCRIPTOR.message_types_by_name['HDF5DataParameter'] = _HDF5DATAPARAMETER -DESCRIPTOR.message_types_by_name['HDF5OutputParameter'] = _HDF5OUTPUTPARAMETER -DESCRIPTOR.message_types_by_name['HingeLossParameter'] = _HINGELOSSPARAMETER -DESCRIPTOR.message_types_by_name['ImageDataParameter'] = _IMAGEDATAPARAMETER -DESCRIPTOR.message_types_by_name['InfogainLossParameter'] = _INFOGAINLOSSPARAMETER -DESCRIPTOR.message_types_by_name['InnerProductParameter'] = _INNERPRODUCTPARAMETER -DESCRIPTOR.message_types_by_name['InputParameter'] = _INPUTPARAMETER -DESCRIPTOR.message_types_by_name['LogParameter'] = _LOGPARAMETER -DESCRIPTOR.message_types_by_name['LRNParameter'] = _LRNPARAMETER -DESCRIPTOR.message_types_by_name['MemoryDataParameter'] = _MEMORYDATAPARAMETER -DESCRIPTOR.message_types_by_name['MVNParameter'] = _MVNPARAMETER -DESCRIPTOR.message_types_by_name['PoolingParameter'] = _POOLINGPARAMETER -DESCRIPTOR.message_types_by_name['PowerParameter'] = _POWERPARAMETER -DESCRIPTOR.message_types_by_name['PythonParameter'] = _PYTHONPARAMETER -DESCRIPTOR.message_types_by_name['ReductionParameter'] = _REDUCTIONPARAMETER -DESCRIPTOR.message_types_by_name['ReLUParameter'] = _RELUPARAMETER -DESCRIPTOR.message_types_by_name['ReshapeParameter'] = _RESHAPEPARAMETER -DESCRIPTOR.message_types_by_name['ScaleParameter'] = _SCALEPARAMETER -DESCRIPTOR.message_types_by_name['SigmoidParameter'] = _SIGMOIDPARAMETER -DESCRIPTOR.message_types_by_name['SliceParameter'] = _SLICEPARAMETER -DESCRIPTOR.message_types_by_name['SoftmaxParameter'] = _SOFTMAXPARAMETER -DESCRIPTOR.message_types_by_name['TanHParameter'] = _TANHPARAMETER -DESCRIPTOR.message_types_by_name['TileParameter'] = _TILEPARAMETER -DESCRIPTOR.message_types_by_name['ThresholdParameter'] = _THRESHOLDPARAMETER -DESCRIPTOR.message_types_by_name['WindowDataParameter'] = _WINDOWDATAPARAMETER -DESCRIPTOR.message_types_by_name['SPPParameter'] = _SPPPARAMETER -DESCRIPTOR.message_types_by_name['V1LayerParameter'] = _V1LAYERPARAMETER -DESCRIPTOR.message_types_by_name['V0LayerParameter'] = _V0LAYERPARAMETER -DESCRIPTOR.message_types_by_name['PReLUParameter'] = _PRELUPARAMETER - -class BlobShape(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _BLOBSHAPE - - # @@protoc_insertion_point(class_scope:caffe.BlobShape) - -class BlobProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _BLOBPROTO - - # @@protoc_insertion_point(class_scope:caffe.BlobProto) - -class BlobProtoVector(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _BLOBPROTOVECTOR - - # @@protoc_insertion_point(class_scope:caffe.BlobProtoVector) - -class Datum(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _DATUM - - # @@protoc_insertion_point(class_scope:caffe.Datum) - -class FillerParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FILLERPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.FillerParameter) - -class NetParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _NETPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.NetParameter) - -class SolverParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SOLVERPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.SolverParameter) - -class SolverState(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SOLVERSTATE - - # @@protoc_insertion_point(class_scope:caffe.SolverState) - -class NetState(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _NETSTATE - - # @@protoc_insertion_point(class_scope:caffe.NetState) - -class NetStateRule(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _NETSTATERULE - - # @@protoc_insertion_point(class_scope:caffe.NetStateRule) - -class ParamSpec(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _PARAMSPEC - - # @@protoc_insertion_point(class_scope:caffe.ParamSpec) - -class LayerParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _LAYERPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.LayerParameter) - -class TransformationParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _TRANSFORMATIONPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.TransformationParameter) - -class LossParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _LOSSPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.LossParameter) - -class AccuracyParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ACCURACYPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.AccuracyParameter) - -class ArgMaxParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ARGMAXPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ArgMaxParameter) - -class ConcatParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CONCATPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ConcatParameter) - -class BatchNormParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _BATCHNORMPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.BatchNormParameter) - -class BiasParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _BIASPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.BiasParameter) - -class ContrastiveLossParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CONTRASTIVELOSSPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ContrastiveLossParameter) - -class ConvolutionParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CONVOLUTIONPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ConvolutionParameter) - -class CropParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CROPPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.CropParameter) - -class DataParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _DATAPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.DataParameter) - -class DropoutParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _DROPOUTPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.DropoutParameter) - -class DummyDataParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _DUMMYDATAPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.DummyDataParameter) - -class EltwiseParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ELTWISEPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.EltwiseParameter) - -class ELUParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ELUPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ELUParameter) - -class EmbedParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _EMBEDPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.EmbedParameter) - -class ExpParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _EXPPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ExpParameter) - -class FlattenParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FLATTENPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.FlattenParameter) - -class HDF5DataParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _HDF5DATAPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.HDF5DataParameter) - -class HDF5OutputParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _HDF5OUTPUTPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.HDF5OutputParameter) - -class HingeLossParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _HINGELOSSPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.HingeLossParameter) - -class ImageDataParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _IMAGEDATAPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ImageDataParameter) - -class InfogainLossParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _INFOGAINLOSSPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.InfogainLossParameter) - -class InnerProductParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _INNERPRODUCTPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.InnerProductParameter) - -class InputParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _INPUTPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.InputParameter) - -class LogParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _LOGPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.LogParameter) - -class LRNParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _LRNPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.LRNParameter) - -class MemoryDataParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _MEMORYDATAPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.MemoryDataParameter) - -class MVNParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _MVNPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.MVNParameter) - -class PoolingParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _POOLINGPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.PoolingParameter) - -class PowerParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _POWERPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.PowerParameter) - -class PythonParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _PYTHONPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.PythonParameter) - -class ReductionParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _REDUCTIONPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ReductionParameter) - -class ReLUParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RELUPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ReLUParameter) - -class ReshapeParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RESHAPEPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ReshapeParameter) - -class ScaleParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SCALEPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ScaleParameter) - -class SigmoidParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SIGMOIDPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.SigmoidParameter) - -class SliceParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SLICEPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.SliceParameter) - -class SoftmaxParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SOFTMAXPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.SoftmaxParameter) - -class TanHParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _TANHPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.TanHParameter) - -class TileParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _TILEPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.TileParameter) - -class ThresholdParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _THRESHOLDPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.ThresholdParameter) - -class WindowDataParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _WINDOWDATAPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.WindowDataParameter) - -class SPPParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SPPPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.SPPParameter) - -class V1LayerParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _V1LAYERPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.V1LayerParameter) - -class V0LayerParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _V0LAYERPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.V0LayerParameter) - -class PReLUParameter(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _PRELUPARAMETER - - # @@protoc_insertion_point(class_scope:caffe.PReLUParameter) - - -_BLOBSHAPE.fields_by_name['dim'].has_options = True -_BLOBSHAPE.fields_by_name['dim']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -_BLOBPROTO.fields_by_name['data'].has_options = True -_BLOBPROTO.fields_by_name['data']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -_BLOBPROTO.fields_by_name['diff'].has_options = True -_BLOBPROTO.fields_by_name['diff']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -_BLOBPROTO.fields_by_name['double_data'].has_options = True -_BLOBPROTO.fields_by_name['double_data']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -_BLOBPROTO.fields_by_name['double_diff'].has_options = True -_BLOBPROTO.fields_by_name['double_diff']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -# @@protoc_insertion_point(module_scope) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bcaffe.proto\x12\x05caffe"\x1c\n\tBlobShape\x12\x0f\n\x03dim\x18\x01 \x03(\x03B\x02\x10\x01"\xcc\x01\n\tBlobProto\x12\x1f\n\x05shape\x18\x07 \x01(\x0b2\x10.caffe.BlobShape\x12\x10\n\x04data\x18\x05 \x03(\x02B\x02\x10\x01\x12\x10\n\x04diff\x18\x06 \x03(\x02B\x02\x10\x01\x12\x17\n\x0bdouble_data\x18\x08 \x03(\x01B\x02\x10\x01\x12\x17\n\x0bdouble_diff\x18\t \x03(\x01B\x02\x10\x01\x12\x0e\n\x03num\x18\x01 \x01(\x05:\x010\x12\x13\n\x08channels\x18\x02 \x01(\x05:\x010\x12\x11\n\x06height\x18\x03 \x01(\x05:\x010\x12\x10\n\x05width\x18\x04 \x01(\x05:\x010"2\n\x0fBlobProtoVector\x12\x1f\n\x05blobs\x18\x01 \x03(\x0b2\x10.caffe.BlobProto"\x81\x01\n\x05Datum\x12\x10\n\x08channels\x18\x01 \x01(\x05\x12\x0e\n\x06height\x18\x02 \x01(\x05\x12\r\n\x05width\x18\x03 \x01(\x05\x12\x0c\n\x04data\x18\x04 \x01(\x0c\x12\r\n\x05label\x18\x05 \x01(\x05\x12\x12\n\nfloat_data\x18\x06 \x03(\x02\x12\x16\n\x07encoded\x18\x07 \x01(\x08:\x05false"\x8a\x02\n\x0fFillerParameter\x12\x16\n\x04type\x18\x01 \x01(\t:\x08constant\x12\x10\n\x05value\x18\x02 \x01(\x02:\x010\x12\x0e\n\x03min\x18\x03 \x01(\x02:\x010\x12\x0e\n\x03max\x18\x04 \x01(\x02:\x011\x12\x0f\n\x04mean\x18\x05 \x01(\x02:\x010\x12\x0e\n\x03std\x18\x06 \x01(\x02:\x011\x12\x12\n\x06sparse\x18\x07 \x01(\x05:\x02-1\x12B\n\rvariance_norm\x18\x08 \x01(\x0e2#.caffe.FillerParameter.VarianceNorm:\x06FAN_IN"4\n\x0cVarianceNorm\x12\n\n\x06FAN_IN\x10\x00\x12\x0b\n\x07FAN_OUT\x10\x01\x12\x0b\n\x07AVERAGE\x10\x02"\x8e\x02\n\x0cNetParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x03 \x03(\t\x12%\n\x0binput_shape\x18\x08 \x03(\x0b2\x10.caffe.BlobShape\x12\x11\n\tinput_dim\x18\x04 \x03(\x05\x12\x1d\n\x0eforce_backward\x18\x05 \x01(\x08:\x05false\x12\x1e\n\x05state\x18\x06 \x01(\x0b2\x0f.caffe.NetState\x12\x19\n\ndebug_info\x18\x07 \x01(\x08:\x05false\x12$\n\x05layer\x18d \x03(\x0b2\x15.caffe.LayerParameter\x12\'\n\x06layers\x18\x02 \x03(\x0b2\x17.caffe.V1LayerParameter"\x9c\n\n\x0fSolverParameter\x12\x0b\n\x03net\x18\x18 \x01(\t\x12&\n\tnet_param\x18\x19 \x01(\x0b2\x13.caffe.NetParameter\x12\x11\n\ttrain_net\x18\x01 \x01(\t\x12\x10\n\x08test_net\x18\x02 \x03(\t\x12,\n\x0ftrain_net_param\x18\x15 \x01(\x0b2\x13.caffe.NetParameter\x12+\n\x0etest_net_param\x18\x16 \x03(\x0b2\x13.caffe.NetParameter\x12$\n\x0btrain_state\x18\x1a \x01(\x0b2\x0f.caffe.NetState\x12#\n\ntest_state\x18\x1b \x03(\x0b2\x0f.caffe.NetState\x12\x11\n\ttest_iter\x18\x03 \x03(\x05\x12\x18\n\rtest_interval\x18\x04 \x01(\x05:\x010\x12 \n\x11test_compute_loss\x18\x13 \x01(\x08:\x05false\x12!\n\x13test_initialization\x18 \x01(\x08:\x04true\x12\x0f\n\x07base_lr\x18\x05 \x01(\x02\x12\x0f\n\x07display\x18\x06 \x01(\x05\x12\x17\n\x0caverage_loss\x18! \x01(\x05:\x011\x12\x10\n\x08max_iter\x18\x07 \x01(\x05\x12\x14\n\titer_size\x18$ \x01(\x05:\x011\x12\x11\n\tlr_policy\x18\x08 \x01(\t\x12\r\n\x05gamma\x18\t \x01(\x02\x12\r\n\x05power\x18\n \x01(\x02\x12\x10\n\x08momentum\x18\x0b \x01(\x02\x12\x14\n\x0cweight_decay\x18\x0c \x01(\x02\x12\x1f\n\x13regularization_type\x18\x1d \x01(\t:\x02L2\x12\x10\n\x08stepsize\x18\r \x01(\x05\x12\x11\n\tstepvalue\x18" \x03(\x05\x12\x1a\n\x0eclip_gradients\x18# \x01(\x02:\x02-1\x12\x13\n\x08snapshot\x18\x0e \x01(\x05:\x010\x12\x17\n\x0fsnapshot_prefix\x18\x0f \x01(\t\x12\x1c\n\rsnapshot_diff\x18\x10 \x01(\x08:\x05false\x12K\n\x0fsnapshot_format\x18% \x01(\x0e2%.caffe.SolverParameter.SnapshotFormat:\x0bBINARYPROTO\x12;\n\x0bsolver_mode\x18\x11 \x01(\x0e2!.caffe.SolverParameter.SolverMode:\x03GPU\x12\x14\n\tdevice_id\x18\x12 \x01(\x05:\x010\x12\x17\n\x0brandom_seed\x18\x14 \x01(\x03:\x02-1\x12\x11\n\x04type\x18( \x01(\t:\x03SGD\x12\x14\n\x05delta\x18\x1f \x01(\x02:\x051e-08\x12\x18\n\tmomentum2\x18\' \x01(\x02:\x050.999\x12\x11\n\trms_decay\x18& \x01(\x02\x12\x19\n\ndebug_info\x18\x17 \x01(\x08:\x05false\x12"\n\x14snapshot_after_train\x18\x1c \x01(\x08:\x04true\x12;\n\x0bsolver_type\x18\x1e \x01(\x0e2!.caffe.SolverParameter.SolverType:\x03SGD"+\n\x0eSnapshotFormat\x12\x08\n\x04HDF5\x10\x00\x12\x0f\n\x0bBINARYPROTO\x10\x01"\x1e\n\nSolverMode\x12\x07\n\x03CPU\x10\x00\x12\x07\n\x03GPU\x10\x01"U\n\nSolverType\x12\x07\n\x03SGD\x10\x00\x12\x0c\n\x08NESTEROV\x10\x01\x12\x0b\n\x07ADAGRAD\x10\x02\x12\x0b\n\x07RMSPROP\x10\x03\x12\x0c\n\x08ADADELTA\x10\x04\x12\x08\n\x04ADAM\x10\x05"l\n\x0bSolverState\x12\x0c\n\x04iter\x18\x01 \x01(\x05\x12\x13\n\x0blearned_net\x18\x02 \x01(\t\x12!\n\x07history\x18\x03 \x03(\x0b2\x10.caffe.BlobProto\x12\x17\n\x0ccurrent_step\x18\x04 \x01(\x05:\x010"N\n\x08NetState\x12!\n\x05phase\x18\x01 \x01(\x0e2\x0c.caffe.Phase:\x04TEST\x12\x10\n\x05level\x18\x02 \x01(\x05:\x010\x12\r\n\x05stage\x18\x03 \x03(\t"s\n\x0cNetStateRule\x12\x1b\n\x05phase\x18\x01 \x01(\x0e2\x0c.caffe.Phase\x12\x11\n\tmin_level\x18\x02 \x01(\x05\x12\x11\n\tmax_level\x18\x03 \x01(\x05\x12\r\n\x05stage\x18\x04 \x03(\t\x12\x11\n\tnot_stage\x18\x05 \x03(\t"\xa3\x01\n\tParamSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x121\n\nshare_mode\x18\x02 \x01(\x0e2\x1d.caffe.ParamSpec.DimCheckMode\x12\x12\n\x07lr_mult\x18\x03 \x01(\x02:\x011\x12\x15\n\ndecay_mult\x18\x04 \x01(\x02:\x011"*\n\x0cDimCheckMode\x12\n\n\x06STRICT\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01"\x98\x13\n\x0eLayerParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06bottom\x18\x03 \x03(\t\x12\x0b\n\x03top\x18\x04 \x03(\t\x12\x1b\n\x05phase\x18\n \x01(\x0e2\x0c.caffe.Phase\x12\x13\n\x0bloss_weight\x18\x05 \x03(\x02\x12\x1f\n\x05param\x18\x06 \x03(\x0b2\x10.caffe.ParamSpec\x12\x1f\n\x05blobs\x18\x07 \x03(\x0b2\x10.caffe.BlobProto\x12\x16\n\x0epropagate_down\x18\x0b \x03(\x08\x12$\n\x07include\x18\x08 \x03(\x0b2\x13.caffe.NetStateRule\x12$\n\x07exclude\x18\t \x03(\x0b2\x13.caffe.NetStateRule\x127\n\x0ftransform_param\x18d \x01(\x0b2\x1e.caffe.TransformationParameter\x12(\n\nloss_param\x18e \x01(\x0b2\x14.caffe.LossParameter\x120\n\x0eaccuracy_param\x18f \x01(\x0b2\x18.caffe.AccuracyParameter\x12,\n\x0cargmax_param\x18g \x01(\x0b2\x16.caffe.ArgMaxParameter\x124\n\x10batch_norm_param\x18\x8b\x01 \x01(\x0b2\x19.caffe.BatchNormParameter\x12)\n\nbias_param\x18\x8d\x01 \x01(\x0b2\x14.caffe.BiasParameter\x12,\n\x0cconcat_param\x18h \x01(\x0b2\x16.caffe.ConcatParameter\x12?\n\x16contrastive_loss_param\x18i \x01(\x0b2\x1f.caffe.ContrastiveLossParameter\x126\n\x11convolution_param\x18j \x01(\x0b2\x1b.caffe.ConvolutionParameter\x12)\n\ncrop_param\x18\x90\x01 \x01(\x0b2\x14.caffe.CropParameter\x12(\n\ndata_param\x18k \x01(\x0b2\x14.caffe.DataParameter\x12.\n\rdropout_param\x18l \x01(\x0b2\x17.caffe.DropoutParameter\x123\n\x10dummy_data_param\x18m \x01(\x0b2\x19.caffe.DummyDataParameter\x12.\n\reltwise_param\x18n \x01(\x0b2\x17.caffe.EltwiseParameter\x12\'\n\telu_param\x18\x8c\x01 \x01(\x0b2\x13.caffe.ELUParameter\x12+\n\x0bembed_param\x18\x89\x01 \x01(\x0b2\x15.caffe.EmbedParameter\x12&\n\texp_param\x18o \x01(\x0b2\x13.caffe.ExpParameter\x12/\n\rflatten_param\x18\x87\x01 \x01(\x0b2\x17.caffe.FlattenParameter\x121\n\x0fhdf5_data_param\x18p \x01(\x0b2\x18.caffe.HDF5DataParameter\x125\n\x11hdf5_output_param\x18q \x01(\x0b2\x1a.caffe.HDF5OutputParameter\x123\n\x10hinge_loss_param\x18r \x01(\x0b2\x19.caffe.HingeLossParameter\x123\n\x10image_data_param\x18s \x01(\x0b2\x19.caffe.ImageDataParameter\x129\n\x13infogain_loss_param\x18t \x01(\x0b2\x1c.caffe.InfogainLossParameter\x129\n\x13inner_product_param\x18u \x01(\x0b2\x1c.caffe.InnerProductParameter\x12+\n\x0binput_param\x18\x8f\x01 \x01(\x0b2\x15.caffe.InputParameter\x12\'\n\tlog_param\x18\x86\x01 \x01(\x0b2\x13.caffe.LogParameter\x12&\n\tlrn_param\x18v \x01(\x0b2\x13.caffe.LRNParameter\x125\n\x11memory_data_param\x18w \x01(\x0b2\x1a.caffe.MemoryDataParameter\x12&\n\tmvn_param\x18x \x01(\x0b2\x13.caffe.MVNParameter\x12.\n\rpooling_param\x18y \x01(\x0b2\x17.caffe.PoolingParameter\x12*\n\x0bpower_param\x18z \x01(\x0b2\x15.caffe.PowerParameter\x12+\n\x0bprelu_param\x18\x83\x01 \x01(\x0b2\x15.caffe.PReLUParameter\x12-\n\x0cpython_param\x18\x82\x01 \x01(\x0b2\x16.caffe.PythonParameter\x123\n\x0freduction_param\x18\x88\x01 \x01(\x0b2\x19.caffe.ReductionParameter\x12(\n\nrelu_param\x18{ \x01(\x0b2\x14.caffe.ReLUParameter\x12/\n\rreshape_param\x18\x85\x01 \x01(\x0b2\x17.caffe.ReshapeParameter\x12+\n\x0bscale_param\x18\x8e\x01 \x01(\x0b2\x15.caffe.ScaleParameter\x12.\n\rsigmoid_param\x18| \x01(\x0b2\x17.caffe.SigmoidParameter\x12.\n\rsoftmax_param\x18} \x01(\x0b2\x17.caffe.SoftmaxParameter\x12\'\n\tspp_param\x18\x84\x01 \x01(\x0b2\x13.caffe.SPPParameter\x12*\n\x0bslice_param\x18~ \x01(\x0b2\x15.caffe.SliceParameter\x12(\n\ntanh_param\x18\x7f \x01(\x0b2\x14.caffe.TanHParameter\x123\n\x0fthreshold_param\x18\x80\x01 \x01(\x0b2\x19.caffe.ThresholdParameter\x12)\n\ntile_param\x18\x8a\x01 \x01(\x0b2\x14.caffe.TileParameter\x126\n\x11window_data_param\x18\x81\x01 \x01(\x0b2\x1a.caffe.WindowDataParameter"\xb6\x01\n\x17TransformationParameter\x12\x10\n\x05scale\x18\x01 \x01(\x02:\x011\x12\x15\n\x06mirror\x18\x02 \x01(\x08:\x05false\x12\x14\n\tcrop_size\x18\x03 \x01(\r:\x010\x12\x11\n\tmean_file\x18\x04 \x01(\t\x12\x12\n\nmean_value\x18\x05 \x03(\x02\x12\x1a\n\x0bforce_color\x18\x06 \x01(\x08:\x05false\x12\x19\n\nforce_gray\x18\x07 \x01(\x08:\x05false"\xc2\x01\n\rLossParameter\x12\x14\n\x0cignore_label\x18\x01 \x01(\x05\x12D\n\rnormalization\x18\x03 \x01(\x0e2&.caffe.LossParameter.NormalizationMode:\x05VALID\x12\x11\n\tnormalize\x18\x02 \x01(\x08"B\n\x11NormalizationMode\x12\x08\n\x04FULL\x10\x00\x12\t\n\x05VALID\x10\x01\x12\x0e\n\nBATCH_SIZE\x10\x02\x12\x08\n\x04NONE\x10\x03"L\n\x11AccuracyParameter\x12\x10\n\x05top_k\x18\x01 \x01(\r:\x011\x12\x0f\n\x04axis\x18\x02 \x01(\x05:\x011\x12\x14\n\x0cignore_label\x18\x03 \x01(\x05"M\n\x0fArgMaxParameter\x12\x1a\n\x0bout_max_val\x18\x01 \x01(\x08:\x05false\x12\x10\n\x05top_k\x18\x02 \x01(\r:\x011\x12\x0c\n\x04axis\x18\x03 \x01(\x05"9\n\x0fConcatParameter\x12\x0f\n\x04axis\x18\x02 \x01(\x05:\x011\x12\x15\n\nconcat_dim\x18\x01 \x01(\r:\x011"j\n\x12BatchNormParameter\x12\x18\n\x10use_global_stats\x18\x01 \x01(\x08\x12&\n\x17moving_average_fraction\x18\x02 \x01(\x02:\x050.999\x12\x12\n\x03eps\x18\x03 \x01(\x02:\x051e-05"]\n\rBiasParameter\x12\x0f\n\x04axis\x18\x01 \x01(\x05:\x011\x12\x13\n\x08num_axes\x18\x02 \x01(\x05:\x011\x12&\n\x06filler\x18\x03 \x01(\x0b2\x16.caffe.FillerParameter"L\n\x18ContrastiveLossParameter\x12\x11\n\x06margin\x18\x01 \x01(\x02:\x011\x12\x1d\n\x0elegacy_version\x18\x02 \x01(\x08:\x05false"\xfc\x03\n\x14ConvolutionParameter\x12\x12\n\nnum_output\x18\x01 \x01(\r\x12\x17\n\tbias_term\x18\x02 \x01(\x08:\x04true\x12\x0b\n\x03pad\x18\x03 \x03(\r\x12\x13\n\x0bkernel_size\x18\x04 \x03(\r\x12\x0e\n\x06stride\x18\x06 \x03(\r\x12\x10\n\x08dilation\x18\x12 \x03(\r\x12\x10\n\x05pad_h\x18\t \x01(\r:\x010\x12\x10\n\x05pad_w\x18\n \x01(\r:\x010\x12\x10\n\x08kernel_h\x18\x0b \x01(\r\x12\x10\n\x08kernel_w\x18\x0c \x01(\r\x12\x10\n\x08stride_h\x18\r \x01(\r\x12\x10\n\x08stride_w\x18\x0e \x01(\r\x12\x10\n\x05group\x18\x05 \x01(\r:\x011\x12-\n\rweight_filler\x18\x07 \x01(\x0b2\x16.caffe.FillerParameter\x12+\n\x0bbias_filler\x18\x08 \x01(\x0b2\x16.caffe.FillerParameter\x12;\n\x06engine\x18\x0f \x01(\x0e2".caffe.ConvolutionParameter.Engine:\x07DEFAULT\x12\x0f\n\x04axis\x18\x10 \x01(\x05:\x011\x12\x1e\n\x0fforce_nd_im2col\x18\x11 \x01(\x08:\x05false"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"0\n\rCropParameter\x12\x0f\n\x04axis\x18\x01 \x01(\x05:\x012\x12\x0e\n\x06offset\x18\x02 \x03(\r"\xa4\x02\n\rDataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x12\n\nbatch_size\x18\x04 \x01(\r\x12\x14\n\trand_skip\x18\x07 \x01(\r:\x010\x121\n\x07backend\x18\x08 \x01(\x0e2\x17.caffe.DataParameter.DB:\x07LEVELDB\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x011\x12\x11\n\tmean_file\x18\x03 \x01(\t\x12\x14\n\tcrop_size\x18\x05 \x01(\r:\x010\x12\x15\n\x06mirror\x18\x06 \x01(\x08:\x05false\x12"\n\x13force_encoded_color\x18\t \x01(\x08:\x05false\x12\x13\n\x08prefetch\x18\n \x01(\r:\x014"\x1b\n\x02DB\x12\x0b\n\x07LEVELDB\x10\x00\x12\x08\n\x04LMDB\x10\x01".\n\x10DropoutParameter\x12\x1a\n\rdropout_ratio\x18\x01 \x01(\x02:\x030.5"\xa0\x01\n\x12DummyDataParameter\x12+\n\x0bdata_filler\x18\x01 \x03(\x0b2\x16.caffe.FillerParameter\x12\x1f\n\x05shape\x18\x06 \x03(\x0b2\x10.caffe.BlobShape\x12\x0b\n\x03num\x18\x02 \x03(\r\x12\x10\n\x08channels\x18\x03 \x03(\r\x12\x0e\n\x06height\x18\x04 \x03(\r\x12\r\n\x05width\x18\x05 \x03(\r"\xa5\x01\n\x10EltwiseParameter\x129\n\toperation\x18\x01 \x01(\x0e2!.caffe.EltwiseParameter.EltwiseOp:\x03SUM\x12\r\n\x05coeff\x18\x02 \x03(\x02\x12\x1e\n\x10stable_prod_grad\x18\x03 \x01(\x08:\x04true"\'\n\tEltwiseOp\x12\x08\n\x04PROD\x10\x00\x12\x07\n\x03SUM\x10\x01\x12\x07\n\x03MAX\x10\x02" \n\x0cELUParameter\x12\x10\n\x05alpha\x18\x01 \x01(\x02:\x011"\xac\x01\n\x0eEmbedParameter\x12\x12\n\nnum_output\x18\x01 \x01(\r\x12\x11\n\tinput_dim\x18\x02 \x01(\r\x12\x17\n\tbias_term\x18\x03 \x01(\x08:\x04true\x12-\n\rweight_filler\x18\x04 \x01(\x0b2\x16.caffe.FillerParameter\x12+\n\x0bbias_filler\x18\x05 \x01(\x0b2\x16.caffe.FillerParameter"D\n\x0cExpParameter\x12\x10\n\x04base\x18\x01 \x01(\x02:\x02-1\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x011\x12\x10\n\x05shift\x18\x03 \x01(\x02:\x010"9\n\x10FlattenParameter\x12\x0f\n\x04axis\x18\x01 \x01(\x05:\x011\x12\x14\n\x08end_axis\x18\x02 \x01(\x05:\x02-1"O\n\x11HDF5DataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x12\n\nbatch_size\x18\x02 \x01(\r\x12\x16\n\x07shuffle\x18\x03 \x01(\x08:\x05false"(\n\x13HDF5OutputParameter\x12\x11\n\tfile_name\x18\x01 \x01(\t"^\n\x12HingeLossParameter\x120\n\x04norm\x18\x01 \x01(\x0e2\x1e.caffe.HingeLossParameter.Norm:\x02L1"\x16\n\x04Norm\x12\x06\n\x02L1\x10\x01\x12\x06\n\x02L2\x10\x02"\x97\x02\n\x12ImageDataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x15\n\nbatch_size\x18\x04 \x01(\r:\x011\x12\x14\n\trand_skip\x18\x07 \x01(\r:\x010\x12\x16\n\x07shuffle\x18\x08 \x01(\x08:\x05false\x12\x15\n\nnew_height\x18\t \x01(\r:\x010\x12\x14\n\tnew_width\x18\n \x01(\r:\x010\x12\x16\n\x08is_color\x18\x0b \x01(\x08:\x04true\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x011\x12\x11\n\tmean_file\x18\x03 \x01(\t\x12\x14\n\tcrop_size\x18\x05 \x01(\r:\x010\x12\x15\n\x06mirror\x18\x06 \x01(\x08:\x05false\x12\x15\n\x0broot_folder\x18\x0c \x01(\t:\x00"\'\n\x15InfogainLossParameter\x12\x0e\n\x06source\x18\x01 \x01(\t"\xcb\x01\n\x15InnerProductParameter\x12\x12\n\nnum_output\x18\x01 \x01(\r\x12\x17\n\tbias_term\x18\x02 \x01(\x08:\x04true\x12-\n\rweight_filler\x18\x03 \x01(\x0b2\x16.caffe.FillerParameter\x12+\n\x0bbias_filler\x18\x04 \x01(\x0b2\x16.caffe.FillerParameter\x12\x0f\n\x04axis\x18\x05 \x01(\x05:\x011\x12\x18\n\ttranspose\x18\x06 \x01(\x08:\x05false"1\n\x0eInputParameter\x12\x1f\n\x05shape\x18\x01 \x03(\x0b2\x10.caffe.BlobShape"D\n\x0cLogParameter\x12\x10\n\x04base\x18\x01 \x01(\x02:\x02-1\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x011\x12\x10\n\x05shift\x18\x03 \x01(\x02:\x010"\xb8\x02\n\x0cLRNParameter\x12\x15\n\nlocal_size\x18\x01 \x01(\r:\x015\x12\x10\n\x05alpha\x18\x02 \x01(\x02:\x011\x12\x12\n\x04beta\x18\x03 \x01(\x02:\x040.75\x12D\n\x0bnorm_region\x18\x04 \x01(\x0e2\x1e.caffe.LRNParameter.NormRegion:\x0fACROSS_CHANNELS\x12\x0c\n\x01k\x18\x05 \x01(\x02:\x011\x123\n\x06engine\x18\x06 \x01(\x0e2\x1a.caffe.LRNParameter.Engine:\x07DEFAULT"5\n\nNormRegion\x12\x13\n\x0fACROSS_CHANNELS\x10\x00\x12\x12\n\x0eWITHIN_CHANNEL\x10\x01"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"Z\n\x13MemoryDataParameter\x12\x12\n\nbatch_size\x18\x01 \x01(\r\x12\x10\n\x08channels\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\r\n\x05width\x18\x04 \x01(\r"d\n\x0cMVNParameter\x12 \n\x12normalize_variance\x18\x01 \x01(\x08:\x04true\x12\x1e\n\x0facross_channels\x18\x02 \x01(\x08:\x05false\x12\x12\n\x03eps\x18\x03 \x01(\x02:\x051e-09"\xa2\x03\n\x10PoolingParameter\x125\n\x04pool\x18\x01 \x01(\x0e2".caffe.PoolingParameter.PoolMethod:\x03MAX\x12\x0e\n\x03pad\x18\x04 \x01(\r:\x010\x12\x10\n\x05pad_h\x18\t \x01(\r:\x010\x12\x10\n\x05pad_w\x18\n \x01(\r:\x010\x12\x13\n\x0bkernel_size\x18\x02 \x01(\r\x12\x10\n\x08kernel_h\x18\x05 \x01(\r\x12\x10\n\x08kernel_w\x18\x06 \x01(\r\x12\x11\n\x06stride\x18\x03 \x01(\r:\x011\x12\x10\n\x08stride_h\x18\x07 \x01(\r\x12\x10\n\x08stride_w\x18\x08 \x01(\r\x127\n\x06engine\x18\x0b \x01(\x0e2\x1e.caffe.PoolingParameter.Engine:\x07DEFAULT\x12\x1d\n\x0eglobal_pooling\x18\x0c \x01(\x08:\x05false".\n\nPoolMethod\x12\x07\n\x03MAX\x10\x00\x12\x07\n\x03AVE\x10\x01\x12\x0e\n\nSTOCHASTIC\x10\x02"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"F\n\x0ePowerParameter\x12\x10\n\x05power\x18\x01 \x01(\x02:\x011\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x011\x12\x10\n\x05shift\x18\x03 \x01(\x02:\x010"g\n\x0fPythonParameter\x12\x0e\n\x06module\x18\x01 \x01(\t\x12\r\n\x05layer\x18\x02 \x01(\t\x12\x13\n\tparam_str\x18\x03 \x01(\t:\x00\x12 \n\x11share_in_parallel\x18\x04 \x01(\x08:\x05false"\xad\x01\n\x12ReductionParameter\x12=\n\toperation\x18\x01 \x01(\x0e2%.caffe.ReductionParameter.ReductionOp:\x03SUM\x12\x0f\n\x04axis\x18\x02 \x01(\x05:\x010\x12\x10\n\x05coeff\x18\x03 \x01(\x02:\x011"5\n\x0bReductionOp\x12\x07\n\x03SUM\x10\x01\x12\x08\n\x04ASUM\x10\x02\x12\t\n\x05SUMSQ\x10\x03\x12\x08\n\x04MEAN\x10\x04"\x8d\x01\n\rReLUParameter\x12\x19\n\x0enegative_slope\x18\x01 \x01(\x02:\x010\x124\n\x06engine\x18\x02 \x01(\x0e2\x1b.caffe.ReLUParameter.Engine:\x07DEFAULT"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"Z\n\x10ReshapeParameter\x12\x1f\n\x05shape\x18\x01 \x01(\x0b2\x10.caffe.BlobShape\x12\x0f\n\x04axis\x18\x02 \x01(\x05:\x010\x12\x14\n\x08num_axes\x18\x03 \x01(\x05:\x02-1"\xa5\x01\n\x0eScaleParameter\x12\x0f\n\x04axis\x18\x01 \x01(\x05:\x011\x12\x13\n\x08num_axes\x18\x02 \x01(\x05:\x011\x12&\n\x06filler\x18\x03 \x01(\x0b2\x16.caffe.FillerParameter\x12\x18\n\tbias_term\x18\x04 \x01(\x08:\x05false\x12+\n\x0bbias_filler\x18\x05 \x01(\x0b2\x16.caffe.FillerParameter"x\n\x10SigmoidParameter\x127\n\x06engine\x18\x01 \x01(\x0e2\x1e.caffe.SigmoidParameter.Engine:\x07DEFAULT"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"L\n\x0eSliceParameter\x12\x0f\n\x04axis\x18\x03 \x01(\x05:\x011\x12\x13\n\x0bslice_point\x18\x02 \x03(\r\x12\x14\n\tslice_dim\x18\x01 \x01(\r:\x011"\x89\x01\n\x10SoftmaxParameter\x127\n\x06engine\x18\x01 \x01(\x0e2\x1e.caffe.SoftmaxParameter.Engine:\x07DEFAULT\x12\x0f\n\x04axis\x18\x02 \x01(\x05:\x011"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"r\n\rTanHParameter\x124\n\x06engine\x18\x01 \x01(\x0e2\x1b.caffe.TanHParameter.Engine:\x07DEFAULT"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"/\n\rTileParameter\x12\x0f\n\x04axis\x18\x01 \x01(\x05:\x011\x12\r\n\x05tiles\x18\x02 \x01(\x05"*\n\x12ThresholdParameter\x12\x14\n\tthreshold\x18\x01 \x01(\x02:\x010"\xc1\x02\n\x13WindowDataParameter\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x10\n\x05scale\x18\x02 \x01(\x02:\x011\x12\x11\n\tmean_file\x18\x03 \x01(\t\x12\x12\n\nbatch_size\x18\x04 \x01(\r\x12\x14\n\tcrop_size\x18\x05 \x01(\r:\x010\x12\x15\n\x06mirror\x18\x06 \x01(\x08:\x05false\x12\x19\n\x0cfg_threshold\x18\x07 \x01(\x02:\x030.5\x12\x19\n\x0cbg_threshold\x18\x08 \x01(\x02:\x030.5\x12\x19\n\x0bfg_fraction\x18\t \x01(\x02:\x040.25\x12\x16\n\x0bcontext_pad\x18\n \x01(\r:\x010\x12\x17\n\tcrop_mode\x18\x0b \x01(\t:\x04warp\x12\x1b\n\x0ccache_images\x18\x0c \x01(\x08:\x05false\x12\x15\n\x0broot_folder\x18\r \x01(\t:\x00"\xeb\x01\n\x0cSPPParameter\x12\x16\n\x0epyramid_height\x18\x01 \x01(\r\x121\n\x04pool\x18\x02 \x01(\x0e2\x1e.caffe.SPPParameter.PoolMethod:\x03MAX\x123\n\x06engine\x18\x06 \x01(\x0e2\x1a.caffe.SPPParameter.Engine:\x07DEFAULT".\n\nPoolMethod\x12\x07\n\x03MAX\x10\x00\x12\x07\n\x03AVE\x10\x01\x12\x0e\n\nSTOCHASTIC\x10\x02"+\n\x06Engine\x12\x0b\n\x07DEFAULT\x10\x00\x12\t\n\x05CAFFE\x10\x01\x12\t\n\x05CUDNN\x10\x02"\xe0\x13\n\x10V1LayerParameter\x12\x0e\n\x06bottom\x18\x02 \x03(\t\x12\x0b\n\x03top\x18\x03 \x03(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12$\n\x07include\x18 \x03(\x0b2\x13.caffe.NetStateRule\x12$\n\x07exclude\x18! \x03(\x0b2\x13.caffe.NetStateRule\x12/\n\x04type\x18\x05 \x01(\x0e2!.caffe.V1LayerParameter.LayerType\x12\x1f\n\x05blobs\x18\x06 \x03(\x0b2\x10.caffe.BlobProto\x12\x0e\n\x05param\x18\xe9\x07 \x03(\t\x12>\n\x0fblob_share_mode\x18\xea\x07 \x03(\x0e2$.caffe.V1LayerParameter.DimCheckMode\x12\x10\n\x08blobs_lr\x18\x07 \x03(\x02\x12\x14\n\x0cweight_decay\x18\x08 \x03(\x02\x12\x13\n\x0bloss_weight\x18# \x03(\x02\x120\n\x0eaccuracy_param\x18\x1b \x01(\x0b2\x18.caffe.AccuracyParameter\x12,\n\x0cargmax_param\x18\x17 \x01(\x0b2\x16.caffe.ArgMaxParameter\x12,\n\x0cconcat_param\x18\t \x01(\x0b2\x16.caffe.ConcatParameter\x12?\n\x16contrastive_loss_param\x18( \x01(\x0b2\x1f.caffe.ContrastiveLossParameter\x126\n\x11convolution_param\x18\n \x01(\x0b2\x1b.caffe.ConvolutionParameter\x12(\n\ndata_param\x18\x0b \x01(\x0b2\x14.caffe.DataParameter\x12.\n\rdropout_param\x18\x0c \x01(\x0b2\x17.caffe.DropoutParameter\x123\n\x10dummy_data_param\x18\x1a \x01(\x0b2\x19.caffe.DummyDataParameter\x12.\n\reltwise_param\x18\x18 \x01(\x0b2\x17.caffe.EltwiseParameter\x12&\n\texp_param\x18) \x01(\x0b2\x13.caffe.ExpParameter\x121\n\x0fhdf5_data_param\x18\r \x01(\x0b2\x18.caffe.HDF5DataParameter\x125\n\x11hdf5_output_param\x18\x0e \x01(\x0b2\x1a.caffe.HDF5OutputParameter\x123\n\x10hinge_loss_param\x18\x1d \x01(\x0b2\x19.caffe.HingeLossParameter\x123\n\x10image_data_param\x18\x0f \x01(\x0b2\x19.caffe.ImageDataParameter\x129\n\x13infogain_loss_param\x18\x10 \x01(\x0b2\x1c.caffe.InfogainLossParameter\x129\n\x13inner_product_param\x18\x11 \x01(\x0b2\x1c.caffe.InnerProductParameter\x12&\n\tlrn_param\x18\x12 \x01(\x0b2\x13.caffe.LRNParameter\x125\n\x11memory_data_param\x18\x16 \x01(\x0b2\x1a.caffe.MemoryDataParameter\x12&\n\tmvn_param\x18" \x01(\x0b2\x13.caffe.MVNParameter\x12.\n\rpooling_param\x18\x13 \x01(\x0b2\x17.caffe.PoolingParameter\x12*\n\x0bpower_param\x18\x15 \x01(\x0b2\x15.caffe.PowerParameter\x12(\n\nrelu_param\x18\x1e \x01(\x0b2\x14.caffe.ReLUParameter\x12.\n\rsigmoid_param\x18& \x01(\x0b2\x17.caffe.SigmoidParameter\x12.\n\rsoftmax_param\x18\' \x01(\x0b2\x17.caffe.SoftmaxParameter\x12*\n\x0bslice_param\x18\x1f \x01(\x0b2\x15.caffe.SliceParameter\x12(\n\ntanh_param\x18% \x01(\x0b2\x14.caffe.TanHParameter\x122\n\x0fthreshold_param\x18\x19 \x01(\x0b2\x19.caffe.ThresholdParameter\x125\n\x11window_data_param\x18\x14 \x01(\x0b2\x1a.caffe.WindowDataParameter\x127\n\x0ftransform_param\x18$ \x01(\x0b2\x1e.caffe.TransformationParameter\x12(\n\nloss_param\x18* \x01(\x0b2\x14.caffe.LossParameter\x12&\n\x05layer\x18\x01 \x01(\x0b2\x17.caffe.V0LayerParameter"\xd8\x04\n\tLayerType\x12\x08\n\x04NONE\x10\x00\x12\n\n\x06ABSVAL\x10#\x12\x0c\n\x08ACCURACY\x10\x01\x12\n\n\x06ARGMAX\x10\x1e\x12\x08\n\x04BNLL\x10\x02\x12\n\n\x06CONCAT\x10\x03\x12\x14\n\x10CONTRASTIVE_LOSS\x10%\x12\x0f\n\x0bCONVOLUTION\x10\x04\x12\x08\n\x04DATA\x10\x05\x12\x11\n\rDECONVOLUTION\x10\'\x12\x0b\n\x07DROPOUT\x10\x06\x12\x0e\n\nDUMMY_DATA\x10 \x12\x12\n\x0eEUCLIDEAN_LOSS\x10\x07\x12\x0b\n\x07ELTWISE\x10\x19\x12\x07\n\x03EXP\x10&\x12\x0b\n\x07FLATTEN\x10\x08\x12\r\n\tHDF5_DATA\x10\t\x12\x0f\n\x0bHDF5_OUTPUT\x10\n\x12\x0e\n\nHINGE_LOSS\x10\x1c\x12\n\n\x06IM2COL\x10\x0b\x12\x0e\n\nIMAGE_DATA\x10\x0c\x12\x11\n\rINFOGAIN_LOSS\x10\r\x12\x11\n\rINNER_PRODUCT\x10\x0e\x12\x07\n\x03LRN\x10\x0f\x12\x0f\n\x0bMEMORY_DATA\x10\x1d\x12\x1d\n\x19MULTINOMIAL_LOGISTIC_LOSS\x10\x10\x12\x07\n\x03MVN\x10"\x12\x0b\n\x07POOLING\x10\x11\x12\t\n\x05POWER\x10\x1a\x12\x08\n\x04RELU\x10\x12\x12\x0b\n\x07SIGMOID\x10\x13\x12\x1e\n\x1aSIGMOID_CROSS_ENTROPY_LOSS\x10\x1b\x12\x0b\n\x07SILENCE\x10$\x12\x0b\n\x07SOFTMAX\x10\x14\x12\x10\n\x0cSOFTMAX_LOSS\x10\x15\x12\t\n\x05SPLIT\x10\x16\x12\t\n\x05SLICE\x10!\x12\x08\n\x04TANH\x10\x17\x12\x0f\n\x0bWINDOW_DATA\x10\x18\x12\r\n\tTHRESHOLD\x10\x1f"*\n\x0cDimCheckMode\x12\n\n\x06STRICT\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01"\xfd\x07\n\x10V0LayerParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x12\n\nnum_output\x18\x03 \x01(\r\x12\x16\n\x08biasterm\x18\x04 \x01(\x08:\x04true\x12-\n\rweight_filler\x18\x05 \x01(\x0b2\x16.caffe.FillerParameter\x12+\n\x0bbias_filler\x18\x06 \x01(\x0b2\x16.caffe.FillerParameter\x12\x0e\n\x03pad\x18\x07 \x01(\r:\x010\x12\x12\n\nkernelsize\x18\x08 \x01(\r\x12\x10\n\x05group\x18\t \x01(\r:\x011\x12\x11\n\x06stride\x18\n \x01(\r:\x011\x125\n\x04pool\x18\x0b \x01(\x0e2".caffe.V0LayerParameter.PoolMethod:\x03MAX\x12\x1a\n\rdropout_ratio\x18\x0c \x01(\x02:\x030.5\x12\x15\n\nlocal_size\x18\r \x01(\r:\x015\x12\x10\n\x05alpha\x18\x0e \x01(\x02:\x011\x12\x12\n\x04beta\x18\x0f \x01(\x02:\x040.75\x12\x0c\n\x01k\x18\x16 \x01(\x02:\x011\x12\x0e\n\x06source\x18\x10 \x01(\t\x12\x10\n\x05scale\x18\x11 \x01(\x02:\x011\x12\x10\n\x08meanfile\x18\x12 \x01(\t\x12\x11\n\tbatchsize\x18\x13 \x01(\r\x12\x13\n\x08cropsize\x18\x14 \x01(\r:\x010\x12\x15\n\x06mirror\x18\x15 \x01(\x08:\x05false\x12\x1f\n\x05blobs\x182 \x03(\x0b2\x10.caffe.BlobProto\x12\x10\n\x08blobs_lr\x183 \x03(\x02\x12\x14\n\x0cweight_decay\x184 \x03(\x02\x12\x14\n\trand_skip\x185 \x01(\r:\x010\x12\x1d\n\x10det_fg_threshold\x186 \x01(\x02:\x030.5\x12\x1d\n\x10det_bg_threshold\x187 \x01(\x02:\x030.5\x12\x1d\n\x0fdet_fg_fraction\x188 \x01(\x02:\x040.25\x12\x1a\n\x0fdet_context_pad\x18: \x01(\r:\x010\x12\x1b\n\rdet_crop_mode\x18; \x01(\t:\x04warp\x12\x12\n\x07new_num\x18< \x01(\x05:\x010\x12\x17\n\x0cnew_channels\x18= \x01(\x05:\x010\x12\x15\n\nnew_height\x18> \x01(\x05:\x010\x12\x14\n\tnew_width\x18? \x01(\x05:\x010\x12\x1d\n\x0eshuffle_images\x18@ \x01(\x08:\x05false\x12\x15\n\nconcat_dim\x18A \x01(\r:\x011\x126\n\x11hdf5_output_param\x18\xe9\x07 \x01(\x0b2\x1a.caffe.HDF5OutputParameter".\n\nPoolMethod\x12\x07\n\x03MAX\x10\x00\x12\x07\n\x03AVE\x10\x01\x12\x0e\n\nSTOCHASTIC\x10\x02"W\n\x0ePReLUParameter\x12&\n\x06filler\x18\x01 \x01(\x0b2\x16.caffe.FillerParameter\x12\x1d\n\x0echannel_shared\x18\x02 \x01(\x08:\x05false*\x1c\n\x05Phase\x12\t\n\x05TRAIN\x10\x00\x12\x08\n\x04TEST\x10\x01') +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, __name__, globals()) diff --git a/imagedt/converters/taffe/caffe/resolver.py b/imagedt/converters/taffe/caffe/resolver.py index 0573d2f..0cf8169 100644 --- a/imagedt/converters/taffe/caffe/resolver.py +++ b/imagedt/converters/taffe/caffe/resolver.py @@ -2,7 +2,8 @@ SHARED_CAFFE_RESOLVER = None -class CaffeResolver(object): + +class CaffeResolver: def __init__(self): self.import_caffe() @@ -12,11 +13,13 @@ def import_caffe(self): # Try to import PyCaffe first # sys.path.append('/data/project/idt-pitaya-serv-train/caffe/python') import caffe + self.caffe = caffe except Exception as e: # Fall back to the protobuf implementation - print e + print(e) from . import caffe_pb2 + self.caffepb = caffe_pb2 show_fallback_warning() if self.caffe: @@ -28,17 +31,20 @@ def import_caffe(self): def has_pycaffe(self): return self.caffe is not None + def get_caffe_resolver(): global SHARED_CAFFE_RESOLVER if SHARED_CAFFE_RESOLVER is None: SHARED_CAFFE_RESOLVER = CaffeResolver() return SHARED_CAFFE_RESOLVER + def has_pycaffe(): return get_caffe_resolver().has_pycaffe() + def show_fallback_warning(): - msg = ''' + msg = """ ------------------------------------------------------------ WARNING: PyCaffe not found! Falling back to a pure protocol buffer implementation. @@ -46,5 +52,5 @@ def show_fallback_warning(): * This backend is UNTESTED! ------------------------------------------------------------ -''' +""" sys.stderr.write(msg) diff --git a/imagedt/converters/taffe/errors.py b/imagedt/converters/taffe/errors.py index f703a44..9ec21bd 100644 --- a/imagedt/converters/taffe/errors.py +++ b/imagedt/converters/taffe/errors.py @@ -1,7 +1,9 @@ import sys + class KaffeError(Exception): pass + def print_stderr(msg): - sys.stderr.write('%s\n' % msg) + sys.stderr.write(f"{msg}\n") diff --git a/imagedt/converters/taffe/graph.py b/imagedt/converters/taffe/graph.py index bec2b3a..64bf9eb 100644 --- a/imagedt/converters/taffe/graph.py +++ b/imagedt/converters/taffe/graph.py @@ -1,12 +1,12 @@ from google.protobuf import text_format from .caffe import get_caffe_resolver -from .errors import KaffeError, print_stderr -from .layers import LayerAdapter, LayerType, NodeKind, NodeDispatch +from .errors import KaffeError +from .layers import LayerAdapter, LayerType, NodeDispatch, NodeKind from .shapes import TensorShape -class Node(object): +class Node: def __init__(self, name, kind, layer=None): self.name = name self.kind = kind @@ -31,8 +31,9 @@ def add_child(self, child_node): def get_only_parent(self): if len(self.parents) != 1: - raise KaffeError('Node (%s) expected to have 1 parent. Found %s.' % - (self, len(self.parents))) + raise KaffeError( + f"Node ({self}) expected to have 1 parent. Found {len(self.parents)}." + ) return self.parents[0] @property @@ -42,14 +43,13 @@ def parameters(self): return None def __str__(self): - return '[%s] %s' % (self.kind, self.name) + return f"[{self.kind}] {self.name}" def __repr__(self): - return '%s (0x%x)' % (self.name, id(self)) + return f"{self.name} (0x{id(self):x})" -class Graph(object): - +class Graph: def __init__(self, nodes=None, name=None): self.nodes = nodes or [] self.node_lut = {node.name: node for node in self.nodes} @@ -62,8 +62,8 @@ def add_node(self, node): def get_node(self, name): try: return self.node_lut[name] - except KeyError: - raise KaffeError('Layer not found: %s' % name) + except KeyError as error: + raise KaffeError(f"Layer not found: {name}") from error def get_input_nodes(self): return [node for node in self.nodes if len(node.parents) == 0] @@ -72,26 +72,23 @@ def get_output_nodes(self): return [node for node in self.nodes if len(node.children) == 0] def topologically_sorted(self): - sorted_nodes = [] - unsorted_nodes = list(self.nodes) - temp_marked = set() - perm_marked = set() - - def visit(node): - if node in temp_marked: - raise KaffeError('Graph is not a DAG.') - if node in perm_marked: - return - temp_marked.add(node) + from collections import deque + + indegrees = {node: len(node.parents) for node in self.nodes} + ready = deque(node for node in self.nodes if not indegrees[node]) + result = [] + while ready: + node = ready.popleft() + result.append(node) for child in node.children: - visit(child) - perm_marked.add(node) - temp_marked.remove(node) - sorted_nodes.insert(0, node) - - while len(unsorted_nodes): - visit(unsorted_nodes.pop()) - return sorted_nodes + if child not in indegrees: + raise KaffeError("Graph contains a child outside its node list") + indegrees[child] -= 1 + if not indegrees[child]: + ready.append(child) + if len(result) != len(self.nodes): + raise KaffeError("Graph is not a DAG.") + return result def compute_output_shapes(self): sorted_nodes = self.topologically_sorted() @@ -106,7 +103,7 @@ def transformed(self, transformers): for transformer in transformers: graph = transformer(graph) if graph is None: - raise KaffeError('Transformer failed: {}'.format(transformer)) + raise KaffeError(f"Transformer failed: {transformer}") assert isinstance(graph, Graph) return graph @@ -114,40 +111,41 @@ def __contains__(self, key): return key in self.node_lut def __str__(self): - hdr = '{:<20} {:<30} {:>20} {:>20}'.format('Type', 'Name', 'Param', 'Output') - s = [hdr, '-' * 94] + hdr = "{:<20} {:<30} {:>20} {:>20}".format("Type", "Name", "Param", "Output") + s = [hdr, "-" * 94] for node in self.topologically_sorted(): # If the node has learned parameters, display the first one's shape. # In case of convolutions, this corresponds to the weights. - data_shape = node.data[0].shape if node.data else '--' - out_shape = node.output_shape or '--' - s.append('{:<20} {:<30} {:>20} {:>20}'.format(node.kind, node.name, data_shape, - tuple(out_shape))) - return '\n'.join(s) + data_shape = node.data[0].shape if node.data else "--" + out_shape = node.output_shape or "--" + s.append( + f"{node.kind:<20} {node.name:<30} {str(data_shape):>20} {str(tuple(out_shape)):>20}" + ) + return "\n".join(s) -class GraphBuilder(object): - '''Constructs a model graph from a Caffe protocol buffer definition.''' +class GraphBuilder: + """Constructs a model graph from a Caffe protocol buffer definition.""" - def __init__(self, def_path, phase='test'): - ''' + def __init__(self, def_path, phase="test"): + """ def_path: Path to the model definition (.prototxt) data_path: Path to the model data (.caffemodel) phase: Either 'test' or 'train'. Used for filtering phase-specific nodes. - ''' + """ self.def_path = def_path self.phase = phase self.load() def load(self): - '''Load the layer definitions from the prototxt.''' + """Load the layer definitions from the prototxt.""" self.params = get_caffe_resolver().NetParameter() - with open(self.def_path, 'rb') as def_file: + with open(self.def_path, "rb") as def_file: text_format.Merge(def_file.read(), self.params) def filter_layers(self, layers): - '''Filter out layers based on the current phase.''' - phase_map = {0: 'train', 1: 'test'} + """Filter out layers based on the current phase.""" + phase_map = {0: "train", 1: "test"} filtered_layer_names = set() filtered_layers = [] for layer in layers: @@ -155,13 +153,13 @@ def filter_layers(self, layers): if len(layer.include): phase = phase_map[layer.include[0].phase] if len(layer.exclude): - phase = phase_map[1 - layer.include[0].phase] - exclude = (phase != self.phase) + phase = phase_map[1 - layer.exclude[0].phase] + exclude = phase != self.phase # Dropout layers appear in a fair number of Caffe # test-time networks. These are just ignored. We'll # filter them out here. - if (not exclude) and (phase == 'test'): - exclude = (layer.type == LayerType.Dropout) + if (not exclude) and (phase == "test"): + exclude = layer.type == LayerType.Dropout if not exclude: filtered_layers.append(layer) # Guard against dupes. @@ -170,39 +168,39 @@ def filter_layers(self, layers): return filtered_layers def make_node(self, layer): - '''Create a graph node for the given layer.''' + """Create a graph node for the given layer.""" kind = NodeKind.map_raw_kind(layer.type) if kind is None: - raise KaffeError('Unknown layer type encountered: %s' % layer.type) + raise KaffeError(f"Unknown layer type encountered: {layer.type}") # We want to use the layer's top names (the "output" names), rather than the # name attribute, which is more of readability thing than a functional one. # Other layers will refer to a node by its "top name". return Node(layer.name, kind, layer=layer) def make_input_nodes(self): - ''' + """ Create data input nodes. This method is for old-style inputs, where the input specification was not treated as a first-class layer in the prototext. Newer models use the "Input layer" type. - ''' + """ nodes = [Node(name, NodeKind.Data) for name in self.params.input] if len(nodes): - input_dim = map(int, self.params.input_dim) + input_dim = list(map(int, self.params.input_dim)) if not input_dim: if len(self.params.input_shape) > 0: - input_dim = map(int, self.params.input_shape[0].dim) + input_dim = list(map(int, self.params.input_shape[0].dim)) else: - raise KaffeError('Dimensions for input not specified.') + raise KaffeError("Dimensions for input not specified.") for node in nodes: node.output_shape = tuple(input_dim) return nodes def build(self): - ''' + """ Builds the graph from the Caffe layer definitions. - ''' + """ # Get the layers layers = self.params.layers or self.params.layer # Filter out phase-excluded layers @@ -232,8 +230,8 @@ def build(self): if (parent_node is None) or (parent_node == node): parent_node = graph.get_node(input_name) node.add_parent(parent_node) - if len(layer.top)>1: - raise KaffeError('Multiple top nodes are not supported.') + if len(layer.top) > 1: + raise KaffeError("Multiple top nodes are not supported.") for output_name in layer.top: if output_name == layer.name: # Output is named the same as the node. No further action required. @@ -257,7 +255,6 @@ def build(self): class NodeMapper(NodeDispatch): - def __init__(self, graph): self.graph = graph @@ -292,11 +289,11 @@ def map_chain(self, chain): return [self.map_node(node) for node in chain] def map_node(self, node): - map_func = self.get_handler(node.kind, 'map') + map_func = self.get_handler(node.kind, "map") mapped_node = map_func(node) assert mapped_node is not None mapped_node.node = node return mapped_node def commit(self, mapped_chains): - raise NotImplementedError('Must be implemented by subclass.') + raise NotImplementedError("Must be implemented by subclass.") diff --git a/imagedt/converters/taffe/layers.py b/imagedt/converters/taffe/layers.py index c3c5955..1f846d6 100644 --- a/imagedt/converters/taffe/layers.py +++ b/imagedt/converters/taffe/layers.py @@ -1,62 +1,72 @@ -import re import numbers +import re from collections import namedtuple -from .shapes import * +from .errors import KaffeError +from .shapes import ( + shape_concat, + shape_convolution, + shape_data, + shape_identity, + shape_inner_product, + shape_mem_data, + shape_not_implemented, + shape_pool, + shape_scalar, +) LAYER_DESCRIPTORS = { - # Caffe Types - 'AbsVal': shape_identity, - 'Accuracy': shape_scalar, - 'ArgMax': shape_not_implemented, - 'BatchNorm': shape_identity, - 'BNLL': shape_not_implemented, - 'Concat': shape_concat, - 'ContrastiveLoss': shape_scalar, - 'Convolution': shape_convolution, - 'Deconvolution': shape_not_implemented, - 'Data': shape_data, - 'Dropout': shape_identity, - 'DummyData': shape_data, - 'EuclideanLoss': shape_scalar, - 'Eltwise': shape_identity, - 'Exp': shape_identity, - 'Flatten': shape_not_implemented, - 'HDF5Data': shape_data, - 'HDF5Output': shape_identity, - 'HingeLoss': shape_scalar, - 'Im2col': shape_not_implemented, - 'ImageData': shape_data, - 'InfogainLoss': shape_scalar, - 'InnerProduct': shape_inner_product, - 'Input': shape_data, - 'LRN': shape_identity, - 'MemoryData': shape_mem_data, - 'MultinomialLogisticLoss': shape_scalar, - 'MVN': shape_not_implemented, - 'Pooling': shape_pool, - 'Power': shape_identity, - 'ReLU': shape_identity, - 'Scale': shape_identity, - 'Sigmoid': shape_identity, - 'SigmoidCrossEntropyLoss': shape_scalar, - 'Silence': shape_not_implemented, - 'Softmax': shape_identity, - 'SoftmaxWithLoss': shape_scalar, - 'Split': shape_not_implemented, - 'Slice': shape_not_implemented, - 'TanH': shape_identity, - 'WindowData': shape_not_implemented, - 'Threshold': shape_identity, + "AbsVal": shape_identity, + "Accuracy": shape_scalar, + "ArgMax": shape_not_implemented, + "BatchNorm": shape_identity, + "BNLL": shape_not_implemented, + "Concat": shape_concat, + "ContrastiveLoss": shape_scalar, + "Convolution": shape_convolution, + "Deconvolution": shape_not_implemented, + "Data": shape_data, + "Dropout": shape_identity, + "DummyData": shape_data, + "EuclideanLoss": shape_scalar, + "Eltwise": shape_identity, + "Exp": shape_identity, + "Flatten": shape_not_implemented, + "HDF5Data": shape_data, + "HDF5Output": shape_identity, + "HingeLoss": shape_scalar, + "Im2col": shape_not_implemented, + "ImageData": shape_data, + "InfogainLoss": shape_scalar, + "InnerProduct": shape_inner_product, + "Input": shape_data, + "LRN": shape_identity, + "MemoryData": shape_mem_data, + "MultinomialLogisticLoss": shape_scalar, + "MVN": shape_not_implemented, + "Pooling": shape_pool, + "Power": shape_identity, + "ReLU": shape_identity, + "Scale": shape_identity, + "Sigmoid": shape_identity, + "SigmoidCrossEntropyLoss": shape_scalar, + "Silence": shape_not_implemented, + "Softmax": shape_identity, + "SoftmaxWithLoss": shape_scalar, + "Split": shape_not_implemented, + "Slice": shape_not_implemented, + "TanH": shape_identity, + "WindowData": shape_not_implemented, + "Threshold": shape_identity, } -LAYER_TYPES = LAYER_DESCRIPTORS.keys() +LAYER_TYPES = list(LAYER_DESCRIPTORS.keys()) -LayerType = type('LayerType', (), {t: t for t in LAYER_TYPES}) +LayerType = type("LayerType", (), {t: t for t in LAYER_TYPES}) -class NodeKind(LayerType): +class NodeKind(LayerType): @staticmethod def map_raw_kind(kind): if kind in LAYER_TYPES: @@ -68,38 +78,38 @@ def compute_output_shape(node): try: val = LAYER_DESCRIPTORS[node.kind](node) return val - except NotImplementedError: - raise KaffeError('Output shape computation not implemented for type: %s' % node.kind) + except NotImplementedError as error: + raise KaffeError( + f"Output shape computation not implemented for type: {node.kind}" + ) from error class NodeDispatchError(KaffeError): - pass -class NodeDispatch(object): - +class NodeDispatch: @staticmethod def get_handler_name(node_kind): if len(node_kind) <= 4: # A catch-all for things like ReLU and tanh return node_kind.lower() # Convert from CamelCase to under_scored - name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', node_kind) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower() + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", node_kind) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() def get_handler(self, node_kind, prefix): name = self.get_handler_name(node_kind) - name = '_'.join((prefix, name)) + name = "_".join((prefix, name)) try: return getattr(self, name) - except AttributeError: - raise NodeDispatchError('No handler found for node kind: %s (expected: %s)' % - (node_kind, name)) - + except AttributeError as error: + raise NodeDispatchError( + f"No handler found for node kind: {node_kind} (expected: {name})" + ) from error -class LayerAdapter(object): +class LayerAdapter: def __init__(self, layer, kind): self.layer = layer self.kind = kind @@ -107,11 +117,13 @@ def __init__(self, layer, kind): @property def parameters(self): name = NodeDispatch.get_handler_name(self.kind) - name = '_'.join((name, 'param')) + name = "_".join((name, "param")) try: return getattr(self.layer, name) - except AttributeError: - raise NodeDispatchError('Caffe parameters not found for layer kind: %s' % (self.kind)) + except AttributeError as error: + raise NodeDispatchError( + f"Caffe parameters not found for layer kind: {self.kind}" + ) from error @staticmethod def get_kernel_value(scalar, repeated, idx, default=None): @@ -127,7 +139,7 @@ def get_kernel_value(scalar, repeated, idx, default=None): # Extract the value for the given spatial dimension return repeated[idx] if default is None: - raise ValueError('Unable to determine kernel parameter!') + raise ValueError("Unable to determine kernel parameter!") return default @property @@ -143,5 +155,7 @@ def kernel_parameters(self): return KernelParameters(k_h, k_w, s_h, s_w, p_h, p_w) -KernelParameters = namedtuple('KernelParameters', ['kernel_h', 'kernel_w', 'stride_h', 'stride_w', - 'pad_h', 'pad_w']) +KernelParameters = namedtuple( + "KernelParameters", + ["kernel_h", "kernel_w", "stride_h", "stride_w", "pad_h", "pad_w"], +) diff --git a/imagedt/converters/taffe/shapes.py b/imagedt/converters/taffe/shapes.py index a70ff14..3cfcf68 100644 --- a/imagedt/converters/taffe/shapes.py +++ b/imagedt/converters/taffe/shapes.py @@ -3,7 +3,7 @@ from .errors import KaffeError -TensorShape = namedtuple('TensorShape', ['batch_size', 'channels', 'height', 'width']) +TensorShape = namedtuple("TensorShape", ["batch_size", "channels", "height", "width"]) def get_filter_output_shape(i_h, i_w, params, round_func): @@ -15,10 +15,11 @@ def get_filter_output_shape(i_h, i_w, params, round_func): def get_strided_kernel_output_shape(node, round_func): assert node.layer is not None input_shape = node.get_only_parent().output_shape - o_h, o_w = get_filter_output_shape(input_shape.height, input_shape.width, - node.layer.kernel_parameters, round_func) + o_h, o_w = get_filter_output_shape( + input_shape.height, input_shape.width, node.layer.kernel_parameters, round_func + ) params = node.layer.parameters - has_c_o = hasattr(params, 'num_output') + has_c_o = hasattr(params, "num_output") c = params.num_output if has_c_o else input_shape.channels return TensorShape(input_shape.batch_size, c, o_h, o_w) @@ -42,16 +43,18 @@ def shape_data(node): return node.output_shape try: # New-style input specification - return map(int, node.parameters.shape[0].dim) - except: + return TensorShape(*map(int, node.parameters.shape[0].dim)) + except (AttributeError, IndexError, TypeError, ValueError) as error: # We most likely have a data layer on our hands. The problem is, # Caffe infers the dimensions of the data from the source (eg: LMDB). # We want to avoid reading datasets here. Fail for now. # This can be temporarily fixed by transforming the data layer to # Caffe's "input" layer (as is usually used in the "deploy" version). # TODO: Find a better solution for this. - raise KaffeError('Cannot determine dimensions of data layer.\n' - 'See comments in function shape_data for more info.') + raise KaffeError( + "Cannot determine dimensions of data layer.\n" + "See comments in function shape_data for more info." + ) from error def shape_mem_data(node): @@ -66,6 +69,14 @@ def shape_concat(node): if output_shape is None: output_shape = list(parent.output_shape) else: + if any( + output_shape[index] != parent.output_shape[index] + for index in range(len(output_shape)) + if index != axis % len(output_shape) + ): + raise KaffeError( + "Concat inputs must agree on non-concatenated dimensions" + ) output_shape[axis] += parent.output_shape[axis] return tuple(output_shape) diff --git a/imagedt/converters/taffe/tensorflow/__init__.py b/imagedt/converters/taffe/tensorflow/__init__.py index 0ce9cf9..5ab6b62 100644 --- a/imagedt/converters/taffe/tensorflow/__init__.py +++ b/imagedt/converters/taffe/tensorflow/__init__.py @@ -1,2 +1,2 @@ -from .transformer import TensorFlowTransformer -from .network import Network +from .network import Network as Network +from .transformer import TensorFlowTransformer as TensorFlowTransformer diff --git a/imagedt/converters/taffe/tensorflow/network.py b/imagedt/converters/taffe/tensorflow/network.py index 3f7f023..8cfd5b2 100644 --- a/imagedt/converters/taffe/tensorflow/network.py +++ b/imagedt/converters/taffe/tensorflow/network.py @@ -1,18 +1,22 @@ import numpy as np -import tensorflow as tf -DEFAULT_PADDING = 'SAME' +from imagedt._backends import tensorflow_v1 + +tf = tensorflow_v1() + + +DEFAULT_PADDING = "SAME" def layer(op): - '''Decorator for composable network layers.''' + """Decorator for composable network layers.""" def layer_decorated(self, *args, **kwargs): # Automatically set a name if not provided. - name = kwargs.setdefault('name', self.get_unique_name(op.__name__)) + name = kwargs.setdefault("name", self.get_unique_name(op.__name__)) # Figure out the layer inputs. if len(self.terminals) == 0: - raise RuntimeError('No input variables found for layer %s.' % name) + raise RuntimeError(f"No input variables found for layer {name}.") elif len(self.terminals) == 1: layer_input = self.terminals[0] else: @@ -29,8 +33,7 @@ def layer_decorated(self, *args, **kwargs): return layer_decorated -class Network(object): - +class Network: def __init__(self, inputs, trainable=True): # The input nodes for this network self.inputs = inputs @@ -41,25 +44,25 @@ def __init__(self, inputs, trainable=True): # If true, the resulting variables are set as trainable self.trainable = trainable # Switch variable for dropout - self.use_dropout = tf.placeholder_with_default(tf.constant(1.0), - shape=[], - name='use_dropout') + self.use_dropout = tf.placeholder_with_default( + tf.constant(1.0), shape=[], name="use_dropout" + ) self.setup() def setup(self): - '''Construct the network. ''' - raise NotImplementedError('Must be implemented by the subclass.') + """Construct the network.""" + raise NotImplementedError("Must be implemented by the subclass.") def load(self, data_path, session, ignore_missing=False): - '''Load network weights. + """Load network weights. data_path: The path to the numpy-serialized network weights session: The current TensorFlow session ignore_missing: If true, serialized weights for missing layers are ignored. - ''' - data_dict = np.load(data_path).item() + """ + data_dict = np.load(data_path, allow_pickle=True, encoding="latin1").item() for op_name in data_dict: with tf.variable_scope(op_name, reuse=True): - for param_name, data in data_dict[op_name].iteritems(): + for param_name, data in data_dict[op_name].items(): try: var = tf.get_variable(param_name) session.run(var.assign(data)) @@ -68,76 +71,91 @@ def load(self, data_path, session, ignore_missing=False): raise def feed(self, *args): - '''Set the input(s) for the next operation by replacing the terminal nodes. + """Set the input(s) for the next operation by replacing the terminal nodes. The arguments can be either layer names or the actual layers. - ''' + """ assert len(args) != 0 self.terminals = [] for fed_layer in args: - if isinstance(fed_layer, basestring): + if isinstance(fed_layer, str): try: fed_layer = self.layers[fed_layer] - except KeyError: - raise KeyError('Unknown layer name fed: %s' % fed_layer) + except KeyError as error: + raise KeyError(f"Unknown layer name fed: {fed_layer}") from error self.terminals.append(fed_layer) return self def get_output(self): - '''Returns the current network output.''' + """Returns the current network output.""" return self.terminals[-1] def get_unique_name(self, prefix): - '''Returns an index-suffixed unique name for the given prefix. + """Returns an index-suffixed unique name for the given prefix. This is used for auto-generating layer names based on the type-prefix. - ''' - ident = sum(t.startswith(prefix) for t, _ in self.layers.items()) + 1 - return '%s_%d' % (prefix, ident) + """ + ident = sum(t.startswith(prefix) for t, _ in list(self.layers.items())) + 1 + return f"{prefix}_{ident}" def make_var(self, name, shape): - '''Creates a new TensorFlow variable.''' - return tf.get_variable(name, shape, trainable=self.trainable) + """Creates a new TensorFlow variable.""" + initializer = None + if name in ("variance", "scale"): + initializer = tf.ones_initializer() + elif name in ("mean", "offset", "biases"): + initializer = tf.zeros_initializer() + return tf.get_variable( + name, shape, trainable=self.trainable, initializer=initializer + ) def validate_padding(self, padding): - '''Verifies that the padding is one of the supported ones.''' - assert padding in ('SAME', 'VALID') + """Verifies that the padding is one of the supported ones.""" + assert padding in ("SAME", "VALID") @layer - def conv(self, - input, - k_h, - k_w, - c_o, - s_h, - s_w, - name, - relu=True, - padding=DEFAULT_PADDING, - group=1, - biased=True): + def conv( + self, + input, + k_h, + k_w, + c_o, + s_h, + s_w, + name, + relu=True, + padding=DEFAULT_PADDING, + group=1, + biased=True, + ): # Verify that the padding is acceptable self.validate_padding(padding) # Get the number of channels in the input - c_i = input.get_shape()[-1] + c_i = input.get_shape().as_list()[-1] # Verify that the grouping parameter is valid assert c_i % group == 0 assert c_o % group == 0 + # Convolution for a given input and kernel - convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding) + def convolve(i, k): + return tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding) + with tf.variable_scope(name) as scope: - kernel = self.make_var('weights', shape=[k_h, k_w, c_i / group, c_o]) + kernel = self.make_var("weights", shape=[k_h, k_w, c_i // group, c_o]) if group == 1: # This is the common-case. Convolve the input without any further complications. output = convolve(input, kernel) else: # Split the input into groups and then convolve each of them independently - input_groups = tf.split(3, group, input) - kernel_groups = tf.split(3, group, kernel) - output_groups = [convolve(i, k) for i, k in zip(input_groups, kernel_groups)] + input_groups = tf.split(input, group, axis=3) + kernel_groups = tf.split(kernel, group, axis=3) + output_groups = [ + convolve(i, k) + for i, k in zip(input_groups, kernel_groups, strict=True) + ] # Concatenate the groups - output = tf.concat(3, output_groups) + output = tf.concat(output_groups, axis=3) # Add the biases if biased: - biases = self.make_var('biases', [c_o]) + biases = self.make_var("biases", [c_o]) output = tf.nn.bias_add(output, biases) if relu: # ReLU non-linearity @@ -151,29 +169,30 @@ def relu(self, input, name): @layer def max_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING): self.validate_padding(padding) - return tf.nn.max_pool(input, - ksize=[1, k_h, k_w, 1], - strides=[1, s_h, s_w, 1], - padding=padding, - name=name) + return tf.nn.max_pool( + input, + ksize=[1, k_h, k_w, 1], + strides=[1, s_h, s_w, 1], + padding=padding, + name=name, + ) @layer def avg_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING): self.validate_padding(padding) - return tf.nn.avg_pool(input, - ksize=[1, k_h, k_w, 1], - strides=[1, s_h, s_w, 1], - padding=padding, - name=name) + return tf.nn.avg_pool( + input, + ksize=[1, k_h, k_w, 1], + strides=[1, s_h, s_w, 1], + padding=padding, + name=name, + ) @layer def lrn(self, input, radius, alpha, beta, name, bias=1.0): - return tf.nn.local_response_normalization(input, - depth_radius=radius, - alpha=alpha, - beta=beta, - bias=bias, - name=name) + return tf.nn.local_response_normalization( + input, depth_radius=radius, alpha=alpha, beta=beta, bias=bias, name=name + ) @layer def concat(self, inputs, axis, name): @@ -183,6 +202,14 @@ def concat(self, inputs, axis, name): def add(self, inputs, name): return tf.add_n(inputs, name=name) + @layer + def multiply(self, inputs, name): + return tf.reduce_prod(tf.stack(inputs), axis=0, name=name) + + @layer + def max(self, inputs, name): + return tf.reduce_max(tf.stack(inputs), axis=0, name=name) + @layer def fc(self, input, num_out, name, relu=True): with tf.variable_scope(name) as scope: @@ -194,16 +221,16 @@ def fc(self, input, num_out, name, relu=True): dim *= d feed_in = tf.reshape(input, [-1, dim]) else: - feed_in, dim = (input, input_shape[-1].value) - weights = self.make_var('weights', shape=[dim, num_out]) - biases = self.make_var('biases', [num_out]) + feed_in, dim = (input, input_shape.as_list()[-1]) + weights = self.make_var("weights", shape=[dim, num_out]) + biases = self.make_var("biases", [num_out]) op = tf.nn.relu_layer if relu else tf.nn.xw_plus_b fc = op(feed_in, weights, biases, name=scope.name) return fc @layer def softmax(self, input, name): - input_shape = map(lambda v: v.value, input.get_shape()) + input_shape = input.get_shape().as_list() if len(input_shape) > 2: # For certain models (like NiN), the singleton spatial dimensions # need to be explicitly squeezed, since they're not broadcast-able @@ -211,29 +238,30 @@ def softmax(self, input, name): if input_shape[1] == 1 and input_shape[2] == 1: input = tf.squeeze(input, squeeze_dims=[1, 2]) else: - raise ValueError('Rank 2 tensor input expected for softmax!') + raise ValueError("Rank 2 tensor input expected for softmax!") return tf.nn.softmax(input, name=name) @layer def batch_normalization(self, input, name, scale_offset=True, relu=False): # NOTE: Currently, only inference is supported - with tf.variable_scope(name) as scope: + with tf.variable_scope(name): shape = [input.get_shape()[-1]] if scale_offset: - scale = self.make_var('scale', shape=shape) - offset = self.make_var('offset', shape=shape) + scale = self.make_var("scale", shape=shape) + offset = self.make_var("offset", shape=shape) else: scale, offset = (None, None) output = tf.nn.batch_normalization( input, - mean=self.make_var('mean', shape=shape), - variance=self.make_var('variance', shape=shape), + mean=self.make_var("mean", shape=shape), + variance=self.make_var("variance", shape=shape), offset=offset, scale=scale, # TODO: This is the default Caffe batch norm eps # Get the actual eps from parameters variance_epsilon=1e-5, - name=name) + name=name, + ) if relu: output = tf.nn.relu(output) return output diff --git a/imagedt/converters/taffe/tensorflow/transformer.py b/imagedt/converters/taffe/tensorflow/transformer.py index fc682fb..c471622 100644 --- a/imagedt/converters/taffe/tensorflow/transformer.py +++ b/imagedt/converters/taffe/tensorflow/transformer.py @@ -3,34 +3,51 @@ from ..errors import KaffeError, print_stderr from ..graph import GraphBuilder, NodeMapper from ..layers import NodeKind -from ..transformers import (DataInjector, DataReshaper, NodeRenamer, ReLUFuser, - BatchNormScaleBiasFuser, BatchNormPreprocessor, ParameterNamer) - +from ..transformers import ( + BatchNormPreprocessor, + BatchNormScaleBiasFuser, + DataInjector, + DataReshaper, + NodeRenamer, + ParameterNamer, + ReLUFuser, +) from . import network def get_padding_type(kernel_params, input_shape, output_shape): - '''Translates Caffe's numeric padding to one of ('SAME', 'VALID'). + """Translates Caffe's numeric padding to one of ('SAME', 'VALID'). Caffe supports arbitrary padding values, while TensorFlow only supports 'SAME' and 'VALID' modes. So, not all Caffe paddings can be translated to TensorFlow. There are some subtleties to how the padding edge-cases are handled. These are described here: https://github.com/Yangqing/caffe2/blob/master/caffe2/proto/caffe2_legacy.proto - ''' + """ k_h, k_w, s_h, s_w, p_h, p_w = kernel_params s_o_h = np.ceil(input_shape.height / float(s_h)) s_o_w = np.ceil(input_shape.width / float(s_w)) - if (output_shape.height == s_o_h) and (output_shape.width == s_o_w): - return 'SAME' + same_pad_h = max((s_o_h - 1) * s_h + k_h - input_shape.height, 0) + same_pad_w = max((s_o_w - 1) * s_w + k_w - input_shape.width, 0) + if ( + output_shape.height == s_o_h + and output_shape.width == s_o_w + and same_pad_h == 2 * p_h + and same_pad_w == 2 * p_w + ): + return "SAME" v_o_h = np.ceil((input_shape.height - k_h + 1.0) / float(s_h)) v_o_w = np.ceil((input_shape.width - k_w + 1.0) / float(s_w)) - if (output_shape.height == v_o_h) and (output_shape.width == v_o_w): - return 'VALID' + if ( + p_h == p_w == 0 + and (output_shape.height == v_o_h) + and (output_shape.width == v_o_w) + ): + return "VALID" return None -class TensorFlowNode(object): - '''An intermediate representation for TensorFlow operations.''' +class TensorFlowNode: + """An intermediate representation for TensorFlow operations.""" def __init__(self, op, *args, **kwargs): # A string corresponding to the TensorFlow operation @@ -43,32 +60,31 @@ def __init__(self, op, *args, **kwargs): self.node = None def format(self, arg): - '''Returns a string representation for the given value.''' - return "'%s'" % arg if isinstance(arg, basestring) else str(arg) + """Returns a string representation for the given value.""" + return repr(arg) def pair(self, key, value): - '''Returns key=formatted(value).''' - return '%s=%s' % (key, self.format(value)) + """Returns key=formatted(value).""" + return f"{key}={self.format(value)}" def emit(self): - '''Emits the Python source for this node.''' + """Emits the Python source for this node.""" # Format positional arguments - args = map(self.format, self.args) + args = list(map(self.format, self.args)) # Format any keyword arguments if self.kwargs: args += [self.pair(k, v) for k, v in self.kwargs] # Set the node name - args.append(self.pair('name', self.node.name)) - args = ', '.join(args) - return '%s(%s)' % (self.op, args) - + args.append(self.pair("name", self.node.name)) + args = ", ".join(args) + return f"{self.op}({args})" -class MaybeActivated(object): +class MaybeActivated: def __init__(self, node, default=True): self.inject_kwargs = {} - if node.metadata.get('relu', False) != default: - self.inject_kwargs['relu'] = not default + if node.metadata.get("relu", False) != default: + self.inject_kwargs["relu"] = not default def __call__(self, *args, **kwargs): kwargs.update(self.inject_kwargs) @@ -76,13 +92,14 @@ def __call__(self, *args, **kwargs): class TensorFlowMapper(NodeMapper): - def get_kernel_params(self, node): kernel_params = node.layer.kernel_parameters input_shape = node.get_only_parent().output_shape padding = get_padding_type(kernel_params, input_shape, node.output_shape) + if padding is None: + raise KaffeError("Caffe padding cannot be represented as SAME or VALID") # Only emit the padding if it's not the default value. - padding = {'padding': padding} if padding != network.DEFAULT_PADDING else {} + padding = {"padding": padding} if padding != network.DEFAULT_PADDING else {} return (kernel_params, padding) def map_convolution(self, node): @@ -90,42 +107,55 @@ def map_convolution(self, node): h = kernel_params.kernel_h w = kernel_params.kernel_w c_o = node.output_shape[1] - c_i = node.parents[0].output_shape[1] + node.parents[0].output_shape[1] group = node.parameters.group if group != 1: - kwargs['group'] = group + kwargs["group"] = group if not node.parameters.bias_term: - kwargs['biased'] = False + kwargs["biased"] = False assert kernel_params.kernel_h == h assert kernel_params.kernel_w == w - return MaybeActivated(node)('conv', kernel_params.kernel_h, kernel_params.kernel_w, c_o, - kernel_params.stride_h, kernel_params.stride_w, **kwargs) + return MaybeActivated(node)( + "conv", + kernel_params.kernel_h, + kernel_params.kernel_w, + c_o, + kernel_params.stride_h, + kernel_params.stride_w, + **kwargs, + ) def map_relu(self, node): - return TensorFlowNode('relu') + return TensorFlowNode("relu") def map_pooling(self, node): pool_type = node.parameters.pool if pool_type == 0: - pool_op = 'max_pool' + pool_op = "max_pool" elif pool_type == 1: - pool_op = 'avg_pool' + pool_op = "avg_pool" else: # Stochastic pooling, for instance. - raise KaffeError('Unsupported pooling type.') + raise KaffeError("Unsupported pooling type.") (kernel_params, padding) = self.get_kernel_params(node) - return TensorFlowNode(pool_op, kernel_params.kernel_h, kernel_params.kernel_w, - kernel_params.stride_h, kernel_params.stride_w, **padding) + return TensorFlowNode( + pool_op, + kernel_params.kernel_h, + kernel_params.kernel_w, + kernel_params.stride_h, + kernel_params.stride_w, + **padding, + ) def map_inner_product(self, node): - #TODO: Axis + # TODO: Axis assert node.parameters.axis == 1 - #TODO: Unbiased - assert node.parameters.bias_term == True - return MaybeActivated(node)('fc', node.parameters.num_output) + # TODO: Unbiased + assert node.parameters.bias_term + return MaybeActivated(node)("fc", node.parameters.num_output) def map_softmax(self, node): - return TensorFlowNode('softmax') + return TensorFlowNode("softmax") def map_lrn(self, node): params = node.parameters @@ -136,67 +166,72 @@ def map_lrn(self, node): # just scales by alpha (as does Krizhevsky's paper). # We'll account for that here. alpha = params.alpha / float(params.local_size) - return TensorFlowNode('lrn', int(params.local_size / 2), alpha, params.beta) + return TensorFlowNode("lrn", int(params.local_size / 2), alpha, params.beta) def map_concat(self, node): - axis = (2, 3, 1, 0)[node.parameters.axis] - return TensorFlowNode('concat', axis) + axis = (0, 3, 1, 2)[node.parameters.axis] + return TensorFlowNode("concat", axis) def map_dropout(self, node): - return TensorFlowNode('dropout', node.parameters.dropout_ratio) + return TensorFlowNode("dropout", 1 - node.parameters.dropout_ratio) def map_batch_norm(self, node): scale_offset = len(node.data) == 4 - kwargs = {} if scale_offset else {'scale_offset': False} - return MaybeActivated(node, default=False)('batch_normalization', **kwargs) + kwargs = {} if scale_offset else {"scale_offset": False} + return MaybeActivated(node, default=False)("batch_normalization", **kwargs) def map_eltwise(self, node): - operations = {0: 'multiply', 1: 'add', 2: 'max'} + operations = {0: "multiply", 1: "add", 2: "max"} op_code = node.parameters.operation try: return TensorFlowNode(operations[op_code]) - except KeyError: - raise KaffeError('Unknown elementwise operation: {}'.format(op_code)) + except KeyError as error: + raise KaffeError(f"Unknown elementwise operation: {op_code}") from error def commit(self, chains): return chains -class TensorFlowEmitter(object): - +class TensorFlowEmitter: def __init__(self, tab=None): - self.tab = tab or ' ' * 4 - self.prefix = '' + self.tab = tab or " " * 4 + self.prefix = "" def indent(self): self.prefix += self.tab def outdent(self): - self.prefix = self.prefix[:-len(self.tab)] + self.prefix = self.prefix[: -len(self.tab)] def statement(self, s): - return self.prefix + s + '\n' + return self.prefix + s + "\n" def emit_imports(self): - return self.statement('from imagedt.converters.taffe.tensorflow import Network\n') + return self.statement( + "from imagedt.converters.taffe.tensorflow import Network\n" + ) def emit_class_def(self, name): - return self.statement('class %s(Network):' % (name)) + return self.statement(f"class {name}(Network):") def emit_setup_def(self): - return self.statement('def setup(self):') + return self.statement("def setup(self):") def emit_parents(self, chain): assert len(chain) - s = '(self.feed(' - sep = ', \n' + self.prefix + (' ' * len(s)) - s += sep.join(["'%s'" % parent.name for parent in chain[0].node.parents]) - return self.statement(s + ')') + s = "(self.feed(" + sep = ", \n" + self.prefix + (" " * len(s)) + s += sep.join([repr(parent.name) for parent in chain[0].node.parents]) + return self.statement(s + ")") def emit_node(self, node): - return self.statement(' ' * 5 + '.' + node.emit()) + return self.statement(" " * 5 + "." + node.emit()) def emit(self, name, chains): + import keyword + + if not name.isidentifier() or keyword.iskeyword(name): + raise KaffeError("Network name must be a valid Python identifier") s = self.emit_imports() s += self.emit_class_def(name) self.indent() @@ -204,18 +239,17 @@ def emit(self, name, chains): self.indent() blocks = [] for chain in chains: - b = '' + b = "" b += self.emit_parents(chain) for node in chain: b += self.emit_node(node) - blocks.append(b[:-1] + ')') - s = s + '\n\n'.join(blocks) + blocks.append(b[:-1] + ")") + s = s + "\n\n".join(blocks) return s -class TensorFlowTransformer(object): - - def __init__(self, def_path, data_path, verbose=True, phase='test'): +class TensorFlowTransformer: + def __init__(self, def_path, data_path, verbose=True, phase="test"): self.verbose = verbose self.phase = phase self.load(def_path, data_path, phase) @@ -234,18 +268,21 @@ def load(self, def_path, data_path, phase): transformers = [ # Fuse split batch normalization layers BatchNormScaleBiasFuser(), - # Fuse ReLUs # TODO: Move non-linearity application to layer wrapper, allowing # any arbitrary operation to be optionally activated. - ReLUFuser(allowed_parent_types=[NodeKind.Convolution, NodeKind.InnerProduct, - NodeKind.BatchNorm]), - + ReLUFuser( + allowed_parent_types=[ + NodeKind.Convolution, + NodeKind.InnerProduct, + NodeKind.BatchNorm, + ] + ), # Rename nodes # Slashes are used for scoping in TensorFlow. Replace slashes # in node names with underscores. # (Caffe's GoogLeNet implementation uses slashes) - NodeRenamer(lambda node: node.name.replace('/', '_')) + NodeRenamer(lambda node: node.name.replace("/", "_")), ] self.graph = graph.transformed(transformers) @@ -256,24 +293,24 @@ def load(self, def_path, data_path, phase): def transform_data(self): if self.params is None: transformers = [ - # Reshape the parameters to TensorFlow's ordering - DataReshaper({ - # (c_o, c_i, h, w) -> (h, w, c_i, c_o) - NodeKind.Convolution: (2, 3, 1, 0), - - # (c_o, c_i) -> (c_i, c_o) - NodeKind.InnerProduct: (1, 0) - }), - + DataReshaper( + { + # (c_o, c_i, h, w) -> (h, w, c_i, c_o) + NodeKind.Convolution: (2, 3, 1, 0), + # (c_o, c_i) -> (c_i, c_o) + NodeKind.InnerProduct: (1, 0), + } + ), # Pre-process batch normalization data BatchNormPreprocessor(), - # Convert parameters to dictionaries ParameterNamer(), ] self.graph = self.graph.transformed(transformers) - self.params = {node.name: node.data for node in self.graph.nodes if node.data} + self.params = { + node.name: node.data for node in self.graph.nodes if node.data + } return self.params def transform_source(self): @@ -282,6 +319,6 @@ def transform_source(self): chains = mapper.map() emitter = TensorFlowEmitter() if not self.graph.name: - self.graph.name = 'MyNet' + self.graph.name = "MyNet" self.source = emitter.emit(self.graph.name, chains) return self.source diff --git a/imagedt/converters/taffe/transformers.py b/imagedt/converters/taffe/transformers.py index cd8a07d..66f0ff6 100644 --- a/imagedt/converters/taffe/transformers.py +++ b/imagedt/converters/taffe/transformers.py @@ -1,8 +1,8 @@ -''' +""" A collection of graph transforms. A transformer is a callable that accepts a graph and returns a transformed version. -''' +""" import numpy as np @@ -11,10 +11,10 @@ from .layers import NodeKind -class DataInjector(object): - ''' +class DataInjector: + """ Associates parameters loaded from a .caffemodel file with their corresponding nodes. - ''' + """ def __init__(self, def_path, data_path): # The .prototxt file defining the graph @@ -37,13 +37,20 @@ def load(self): def load_using_caffe(self): caffe = get_caffe_resolver().caffe net = caffe.Net(self.def_path, self.data_path, caffe.TEST) - data = lambda blob: blob.data - self.params = [(k, map(data, v)) for k, v in net.params.items()] + + def data(blob): + return blob.data + + self.params = [(k, list(map(data, v))) for k, v in list(net.params.items())] def load_using_pb(self): data = get_caffe_resolver().NetParameter() - data.MergeFromString(open(self.data_path, 'rb').read()) - pair = lambda layer: (layer.name, self.normalize_pb_data(layer)) + with open(self.data_path, "rb") as stream: + data.MergeFromString(stream.read()) + + def pair(layer): + return layer.name, self.normalize_pb_data(layer) + layers = data.layers or data.layer self.params = [pair(layer) for layer in layers if layer.blobs] self.did_use_pb = True @@ -53,7 +60,7 @@ def normalize_pb_data(self, layer): for blob in layer.blobs: if len(blob.shape.dim): dims = blob.shape.dim - c_o, c_i, h, w = map(int, [1] * (4 - len(dims)) + list(dims)) + c_o, c_i, h, w = list(map(int, [1] * (4 - len(dims)) + list(dims))) else: c_o = blob.num c_i = blob.channels @@ -72,11 +79,22 @@ def adjust_parameters(self, node, data): # potential for future issues. # The Caffe-backend does not suffer from this problem. data = list(data) - squeeze_indices = [1] # Squeeze biases. + if node.kind == NodeKind.BatchNorm: + if len(data) != 3 or np.asarray(data[2]).size != 1: + raise KaffeError( + "BatchNorm requires mean, variance and one scalar factor" + ) + return [ + np.asarray(data[0]).reshape(-1), + np.asarray(data[1]).reshape(-1), + float(np.asarray(data[2]).reshape(-1)[0]), + ] + if len(data) > 1: + data[1] = np.asarray(data[1]).reshape(-1) + if node.kind == NodeKind.Scale: + data[0] = np.asarray(data[0]).reshape(-1) if node.kind == NodeKind.InnerProduct: - squeeze_indices.append(0) # Squeeze FC. - for idx in squeeze_indices: - data[idx] = np.squeeze(data[idx]) + data[0] = np.asarray(data[0]).reshape(node.parameters.num_output, -1) return data def __call__(self, graph): @@ -85,17 +103,18 @@ def __call__(self, graph): node = graph.get_node(layer_name) node.data = self.adjust_parameters(node, data) else: - print_stderr('Ignoring parameters for non-existent layer: %s' % layer_name) + print_stderr( + f"Ignoring parameters for non-existent layer: {layer_name}" + ) return graph -class DataReshaper(object): - +class DataReshaper: def __init__(self, mapping, replace=True): # A dictionary mapping NodeKind to the transposed order. self.mapping = mapping # The node kinds eligible for reshaping - self.reshaped_node_types = self.mapping.keys() + self.reshaped_node_types = list(self.mapping.keys()) # If true, the reshaped data will replace the old one. # Otherwise, it's set to the reshaped_data attribute. self.replace = replace @@ -111,8 +130,10 @@ def has_spatial_parent(self, node): def map(self, node_kind): try: return self.mapping[node_kind] - except KeyError: - raise KaffeError('Ordering not found for node kind: {}'.format(node_kind)) + except KeyError as error: + raise KaffeError( + f"Ordering not found for node kind: {node_kind}" + ) from error def __call__(self, graph): for node in graph.nodes: @@ -120,8 +141,8 @@ def __call__(self, graph): continue if node.kind not in self.reshaped_node_types: # Check for 2+ dimensional data - if any(len(tensor.shape) > 1 for tensor in node.data): - print_stderr('Warning: parmaters not reshaped for node: {}'.format(node)) + if any(np.ndim(tensor) > 1 for tensor in node.data): + print_stderr(f"Warning: parmaters not reshaped for node: {node}") continue transpose_order = self.map(node.kind) weights = node.data[0] @@ -131,27 +152,34 @@ def __call__(self, graph): in_shape = node.get_only_parent().output_shape fc_shape = weights.shape output_channels = fc_shape[0] - weights = weights.reshape((output_channels, in_shape.channels, in_shape.height, - in_shape.width)) + weights = weights.reshape( + ( + output_channels, + in_shape.channels, + in_shape.height, + in_shape.width, + ) + ) weights = weights.transpose(self.map(NodeKind.Convolution)) - node.reshaped_data = weights.reshape(fc_shape[transpose_order[0]], - fc_shape[transpose_order[1]]) + node.reshaped_data = weights.reshape( + fc_shape[transpose_order[0]], fc_shape[transpose_order[1]] + ) else: node.reshaped_data = weights.transpose(transpose_order) if self.replace: for node in graph.nodes: - if hasattr(node, 'reshaped_data'): + if hasattr(node, "reshaped_data"): # Set the weights node.data[0] = node.reshaped_data del node.reshaped_data return graph -class SubNodeFuser(object): - ''' +class SubNodeFuser: + """ An abstract helper for merging a single-child with its single-parent. - ''' + """ def __call__(self, graph): nodes = graph.nodes @@ -180,18 +208,18 @@ def __call__(self, graph): return graph.replaced(transformed_nodes) def is_eligible_pair(self, parent, child): - '''Returns true if this parent/child pair is eligible for fusion.''' - raise NotImplementedError('Must be implemented by subclass.') + """Returns true if this parent/child pair is eligible for fusion.""" + raise NotImplementedError("Must be implemented by subclass.") def merge(self, parent, child): - '''Merge the child node into the parent.''' - raise NotImplementedError('Must be implemented by subclass') + """Merge the child node into the parent.""" + raise NotImplementedError("Must be implemented by subclass") class ReLUFuser(SubNodeFuser): - ''' + """ Fuses rectified linear units with their parent nodes. - ''' + """ def __init__(self, allowed_parent_types=None): # Fuse ReLUs when the parent node is one of the given types. @@ -199,36 +227,42 @@ def __init__(self, allowed_parent_types=None): self.allowed_parent_types = allowed_parent_types def is_eligible_pair(self, parent, child): - return ((self.allowed_parent_types is None or parent.kind in self.allowed_parent_types) and - child.kind == NodeKind.ReLU) + return ( + self.allowed_parent_types is None + or parent.kind in self.allowed_parent_types + ) and child.kind == NodeKind.ReLU def merge(self, parent, _): - parent.metadata['relu'] = True + parent.metadata["relu"] = True class BatchNormScaleBiasFuser(SubNodeFuser): - ''' + """ The original batch normalization paper includes two learned - parameters: a scaling factor \gamma and a bias \beta. + parameters: a scaling factor gamma and a bias beta. Caffe's implementation does not include these two. However, it is commonly replicated by adding a scaling+bias layer immidiately after the batch norm. This fuser merges the scaling+bias layer with the batch norm. - ''' + """ def is_eligible_pair(self, parent, child): - return (parent.kind == NodeKind.BatchNorm and child.kind == NodeKind.Scale and - child.parameters.axis == 1 and child.parameters.bias_term == True) + return ( + parent.kind == NodeKind.BatchNorm + and child.kind == NodeKind.Scale + and child.parameters.axis == 1 + and child.parameters.bias_term + ) def merge(self, parent, child): parent.scale_bias_node = child -class BatchNormPreprocessor(object): - ''' +class BatchNormPreprocessor: + """ Prescale batch normalization parameters. Concatenate gamma (scale) and beta (bias) terms if set. - ''' + """ def __call__(self, graph): for node in graph.nodes: @@ -243,18 +277,18 @@ def __call__(self, graph): variance *= scaling_factor # Replace with the updated values node.data = [mean, variance] - if hasattr(node, 'scale_bias_node'): + if hasattr(node, "scale_bias_node"): # Include the scale and bias terms gamma, beta = node.scale_bias_node.data node.data += [gamma, beta] return graph -class NodeRenamer(object): - ''' +class NodeRenamer: + """ Renames nodes in the graph using a given unary function that accepts a node and returns its new name. - ''' + """ def __init__(self, renamer): self.renamer = renamer @@ -262,29 +296,32 @@ def __init__(self, renamer): def __call__(self, graph): for node in graph.nodes: node.name = self.renamer(node) + graph.node_lut = {node.name: node for node in graph.nodes} + if len(graph.node_lut) != len(graph.nodes): + raise KaffeError("Node renaming produced duplicate names") return graph -class ParameterNamer(object): - ''' +class ParameterNamer: + """ Convert layer data arrays to a dictionary mapping parameter names to their values. - ''' + """ def __call__(self, graph): for node in graph.nodes: if node.data is None: continue if node.kind in (NodeKind.Convolution, NodeKind.InnerProduct): - names = ('weights',) + names = ("weights",) if node.parameters.bias_term: - names += ('biases',) + names += ("biases",) elif node.kind == NodeKind.BatchNorm: - names = ('mean', 'variance') + names = ("mean", "variance") if len(node.data) == 4: - names += ('scale', 'offset') + names += ("scale", "offset") else: - print_stderr('WARNING: Unhandled parameters: {}'.format(node.kind)) + print_stderr(f"WARNING: Unhandled parameters: {node.kind}") continue assert len(names) == len(node.data) - node.data = dict(zip(names, node.data)) + node.data = dict(list(zip(names, node.data, strict=True))) return graph diff --git a/imagedt/converters/tfmodel_lite.py b/imagedt/converters/tfmodel_lite.py index 154f992..e9f95ad 100644 --- a/imagedt/converters/tfmodel_lite.py +++ b/imagedt/converters/tfmodel_lite.py @@ -1,306 +1,43 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse - -from tensorflow.contrib.lite.python import lite -from tensorflow.python.tools.freeze_graph import freeze_graph -from tensorflow.contrib.lite.toco import toco_flags_pb2 as _toco_flags_pb2 -from tensorflow.contrib.lite.toco import types_pb2 as _types_pb2 - - -def _parse_array(values): - if values: - return values.split(",") - - -def _parse_int_array(values): - if values: - return [int(val) for val in values.split(",")] - - -def _parse_set(values): - if values: - return set(values.split(",")) - - -def _get_toco_converter(flags): - """Makes a TocoConverter object based on the flags provided. - - Args: - flags: argparse.Namespace object containing TFLite flags. - - Returns: - TocoConverter object. - """ - # Parse input and output arrays. - input_arrays = _parse_array(flags.input_arrays) - input_shapes = None - if flags.input_shapes: - input_shapes_list = [ - _parse_int_array(shape) for shape in flags.input_shapes.split(":") - ] - input_shapes = dict(zip(input_arrays, input_shapes_list)) - output_arrays = _parse_array(flags.output_arrays) - - converter_kwargs = { - "input_arrays": input_arrays, - "input_shapes": input_shapes, - "output_arrays": output_arrays - } - - # Create TocoConverter. - if flags.graph_def_file: - converter_fn = lite.TocoConverter.from_frozen_graph - converter_kwargs["graph_def_file"] = flags.graph_def_file - elif flags.saved_model_dir: - converter_fn = lite.TocoConverter.from_saved_model - converter_kwargs["saved_model_dir"] = flags.saved_model_dir - converter_kwargs["tag_set"] = _parse_set(flags.saved_model_tag_set) - converter_kwargs["signature_key"] = flags.saved_model_signature_key - - return converter_fn(**converter_kwargs) - - -def _convert_model(flags): - """Calls function to convert the TensorFlow model into a TFLite model. - - Args: - flags: argparse.Namespace object. - """ - # Create converter. - converter = _get_toco_converter(flags) - - if flags.inference_type: - converter.inference_type = _types_pb2.IODataType.Value(flags.inference_type) - if flags.inference_input_type: - converter.inference_input_type = _types_pb2.IODataType.Value( - flags.inference_input_type) - if flags.output_format: - converter.output_format = _toco_flags_pb2.FileFormat.Value( - flags.output_format) - - if flags.mean_values and flags.std_dev_values: - input_arrays = _parse_array(flags.input_arrays) - std_dev_values = _parse_int_array(flags.std_dev_values) - mean_values = _parse_int_array(flags.mean_values) - quant_stats = zip(mean_values, std_dev_values) - converter.quantized_input_stats = dict(zip(input_arrays, quant_stats)) - if flags.default_ranges_min and flags.default_ranges_max: - converter.default_ranges_stats = (flags.default_ranges_min, - flags.default_ranges_max) - - if flags.drop_control_dependency: - converter.drop_control_dependency = flags.drop_control_dependency - if flags.reorder_across_fake_quant: - converter.reorder_across_fake_quant = flags.reorder_across_fake_quant - if flags.change_concat_input_ranges: - converter.change_concat_input_ranges = flags.change_concat_input_ranges - if flags.allow_custom_ops: - converter.allow_custom_ops = flags.allow_custom_ops - - # Convert model. - output_data = converter.convert() - with open(flags.output_file, "wb") as f: - f.write(output_data) - - -def _check_flags(flags, unparsed): - """Checks the parsed and unparsed flags to ensure they are valid. - - Raises an error if previously support unparsed flags are found. Raises an - error for parsed flags that don't meet the required conditions. - - Args: - flags: argparse.Namespace object containing TFLite flags. - unparsed: List of unparsed flags. - - Raises: - ValueError: Invalid flags. - """ - - # Check unparsed flags for common mistakes based on previous TOCO. - def _get_message_unparsed(flag, orig_flag, new_flag): - if flag.startswith(orig_flag): - return "\n Use {0} instead of {1}".format(new_flag, orig_flag) - return "" - - if unparsed: - output = "" - for flag in unparsed: - output += _get_message_unparsed(flag, "--input_file", "--graph_def_file") - output += _get_message_unparsed(flag, "--std_value", "--std_dev_values") - output += _get_message_unparsed(flag, "--batch_size", "--input_shapes") - raise ValueError(output) - - # Check that flags are valid. - if flags.graph_def_file and (not flags.input_arrays or - not flags.output_arrays): - raise ValueError("--input_arrays and --output_arrays are required with " - "--graph_def_file") - - if flags.input_shapes: - if not flags.input_arrays: - raise ValueError("--input_shapes must be used with --input_arrays") - if flags.input_shapes.count(":") != flags.input_arrays.count(","): - raise ValueError("--input_shapes and --input_arrays must have the same " - "number of items") - - if flags.std_dev_values or flags.mean_values: - if bool(flags.std_dev_values) != bool(flags.mean_values): - raise ValueError("--std_dev_values and --mean_values must be used " - "together") - if not flags.input_arrays: - raise ValueError("--std_dev_values and --mean_values must be used with " - "--input_arrays") - if (flags.std_dev_values.count(",") != flags.mean_values.count(",") or - flags.std_dev_values.count(",") != flags.input_arrays.count(",")): - raise ValueError("--std_dev_values, --mean_values, and --input_arrays " - "must have the same number of items") - - if bool(flags.default_ranges_min) != bool(flags.default_ranges_max): - raise ValueError("--default_ranges_min and --default_ranges_max must be " - "used together") - - -def convert(graph_def_file, output_file, output_format='TFLITE', inference_type='FLOAT', inference_input_type='FLOAT', - input_array='data', input_shape='1,224,224,3', output_arrays='softmax'): - - """Main in toco_convert.py.""" - parser = argparse.ArgumentParser( - description=("Command line tool to run TensorFlow Lite Optimizing " - "Converter (TOCO).")) - - # Output file flag. - parser.add_argument('-output_file', - "--output_file", - type=str, - default=output_file, - help="Full filepath of the output file.") - - # Input file flags. - input_file_group = parser.add_mutually_exclusive_group(required=False) - input_file_group.add_argument('-graph_def_file', - "--graph_def_file", - type=str, - default=graph_def_file, - help="Full filepath of file containing TensorFlow GraphDef.") - input_file_group.add_argument( - "--saved_model_dir", - type=str, - help="Full filepath of directory containing the SavedModel.") - - # Model format flags. - parser.add_argument('-output_format', - "--output_format", - type=str, - default=output_format, - choices=["TFLITE", "GRAPHVIZ_DOT"], - help="Output file format.") - parser.add_argument('-inference_type', - "--inference_type", - type=str, - choices=["FLOAT", "QUANTIZED_UINT8"], - default=inference_type, - help="Target data type of arrays in the output file.") - parser.add_argument('-inference_input_type', - "--inference_input_type", - type=str, - choices=["FLOAT", "QUANTIZED_UINT8"], - default=inference_input_type, - help=("Target data type of input arrays. Allows for a different type for " - "input arrays in the case of quantization.")) - - # Input and output arrays flags. - parser.add_argument('-input_arrays', - "--input_arrays", - type=str, - default=input_array, - help="Names of the output arrays, comma-separated.") - parser.add_argument('-input_shapes', - "--input_shapes", - type=str, - default=input_shape, - help="Shapes corresponding to --input_arrays, colon-separated.") - parser.add_argument('-output_arrays', - "--output_arrays", - type=str, - default=output_arrays, - help="Names of the output arrays, comma-separated.") - - # SavedModel related flags. - parser.add_argument( - "--saved_model_tag_set", - type=str, - help=("Comma-separated set of tags identifying the MetaGraphDef within " - "the SavedModel to analyze. All tags must be present. " - "(default \"serve\")")) - parser.add_argument( - "--saved_model_signature_key", - type=str, - help=("Key identifying the SignatureDef containing inputs and outputs. " - "(default DEFAULT_SERVING_SIGNATURE_DEF_KEY)")) - - # Quantization flags. - parser.add_argument( - "--std_dev_values", - type=str, - help=("Standard deviation of training data for each input tensor, " - "comma-separated. Used for quantization. (default None)")) - parser.add_argument( - "--mean_values", - type=str, - help=("Mean of training data for each input tensor, comma-separated. " - "Used for quantization. (default None)")) - parser.add_argument( - "--default_ranges_min", - type=int, - help=("Default value for min bound of min/max range values used for all " - "arrays without a specified range, Intended for experimenting with " - "quantization via \"dummy quantization\". (default None)")) - parser.add_argument( - "--default_ranges_max", - type=int, - help=("Default value for max bound of min/max range values used for all " - "arrays without a specified range, Intended for experimenting with " - "quantization via \"dummy quantization\". (default None)")) - - # Graph manipulation flags. - parser.add_argument( - "--drop_control_dependency", - type=bool, - help=("Boolean indicating whether to drop control dependencies silently. " - "This is due to TensorFlow not supporting control dependencies. " - "(default True)")) - parser.add_argument( - "--reorder_across_fake_quant", - type=bool, - help=("Boolean indicating whether to reorder FakeQuant nodes in " - "unexpected locations. Used when the location of the FakeQuant " - "nodes is preventing graph transformations necessary to convert " - "the graph. Results in a graph that differs from the quantized " - "training graph, potentially causing differing arithmetic " - "behavior. (default False)")) - parser.add_argument( - "--change_concat_input_ranges", - type=bool, - help=("Boolean to change behavior of min/max ranges for inputs and " - "outputs of the concat operator for quantized models. Changes the " - "ranges of concat operator overlap when true. (default False)")) - parser.add_argument( - "--allow_custom_ops", - type=bool, - help=("Boolean indicating whether to allow custom operations. When false " - "any unknown operation is an error. When true, custom ops are " - "created for any op that is unknown. The developer will need to " - "provide these to the TensorFlow Lite runtime with a custom " - "resolver. (default False)")) - - # parse prams - tflite_flags = parser.parse_args() - - print("converting tflite model......") - _convert_model(tflite_flags) - print("finished......") +"""Frozen GraphDef to TFLite conversion with the original entry point.""" + +from pathlib import Path + +from imagedt._backends import require + + +def convert( + graph_def_file, + output_file, + output_format="TFLITE", + inference_type="FLOAT", + inference_input_type="FLOAT", + input_array="data", + input_shape="1,224,224,3", + output_arrays="softmax", +): + if output_format != "TFLITE": + raise NotImplementedError( + "GRAPHVIZ_DOT is no longer supported by the modern converter" + ) + if inference_type != "FLOAT" or inference_input_type != "FLOAT": + raise NotImplementedError( + "Legacy uint8 conversion requires calibration data; use a representative dataset with the modern TensorFlow converter" + ) + arrays = [name.strip() for name in input_array.split(",")] + shapes = [ + [int(dim) for dim in shape.split(",")] for shape in input_shape.split(":") + ] + if len(arrays) != len(shapes) or any( + not shape or min(shape) < 1 for shape in shapes + ): + raise ValueError("input arrays and positive shapes must correspond one-to-one") + if not Path(graph_def_file).is_file(): + raise FileNotFoundError(graph_def_file) + tf = require("tensorflow", "tensorflow") + converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( + str(graph_def_file), + arrays, + [name.strip() for name in output_arrays.split(",")], + input_shapes=dict(zip(arrays, shapes, strict=True)), + ) + Path(output_file).write_bytes(converter.convert()) diff --git a/imagedt/converters/tfrecords.py b/imagedt/converters/tfrecords.py index c8af471..369e4b2 100644 --- a/imagedt/converters/tfrecords.py +++ b/imagedt/converters/tfrecords.py @@ -1,79 +1,23 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import os -import cv2 -import sys -import lake -import random -import numpy as np from collections import OrderedDict -import tensorflow as tf - -# import imagedt -from ..image.process import noise_padd - -def int64_feature(values): - """Returns a TF-Feature of int64s. - - Args: - values: A scalar or list of values. - - Returns: - A TF-Feature. - """ - if not isinstance(values, (tuple, list)): - values = [values] - return tf.train.Feature(int64_list=tf.train.Int64List(value=values)) - - -def bytes_feature(values): - """Returns a TF-Feature of bytes. - - Args: - values: A string. - - Returns: - A TF-Feature. - """ - return tf.train.Feature(bytes_list=tf.train.BytesList(value=[values])) +import imagedt as lake +from imagedt._backends import tensorflow_v1 +from imagedt.tensorflow.tools.dataset_util import ( + bytes_feature as bytes_feature, +) +from imagedt.tensorflow.tools.dataset_util import ( + float_feature as float_feature, +) +from imagedt.tensorflow.tools.dataset_util import ( + int64_feature as int64_feature, +) -def float_feature(values): - """Returns a TF-Feature of floats. - - Args: - values: A scalar of list of values. - - Returns: - A TF-Feature. - """ - if not isinstance(values, (tuple, list)): - values = [values] - return tf.train.Feature(float_list=tf.train.FloatList(value=values)) - - - - -class ImageReader(object): - """Helper class that provides TensorFlow image coding utilities.""" - - def __init__(self): - # Initializes function that decodes RGB JPEG data. - self._decode_jpeg_data = tf.placeholder(dtype=tf.string) - self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3) - - def read_image_dims(self, sess, image_data): - image = self.decode_jpeg(sess, image_data) - return image.shape[0], image.shape[1] +# import imagedt +from imagedt.tensorflow.tools.Records import ImageReader as ImageReader +from imagedt.tensorflow.tools.Records import RecordWriter - def decode_jpeg(self, sess, image_data): - image = sess.run(self._decode_jpeg, - feed_dict={self._decode_jpeg_data: image_data}) - assert len(image.shape) == 3 - assert image.shape[2] == 3 - return image +tf = tensorflow_v1() def get_files_dir_dict(all_files): @@ -88,94 +32,36 @@ def get_files_dir_dict(all_files): def get_image_pairs(data_dir): - all_files = lake.dir.loop(data_dir, ['.jpg', '.png']) + all_files = lake.dir.loop(data_dir, [".jpg", ".png"]) return all_files def get_label_dict(dir_dict): - return OrderedDict((label, index+1) for index, label in enumerate(dir_dict)) + return OrderedDict((label, index) for index, label in enumerate(dir_dict)) def write_laebl_lines(root_dir, all_files, label_map): label_lines = [] for cls in label_map: label_lines.append([cls, label_map[cls]]) - lake.file.write_csv(label_lines, os.path.join(root_dir, 'labels.txt')) + lake.file.write_csv(label_lines, os.path.join(root_dir, "labels.txt")) f_lines = [] for f_name in all_files: cls = os.path.basename(os.path.dirname(f_name)) f_lines.append([f_name, label_map[cls]]) - lake.file.write_csv(f_lines, os.path.join(root_dir, 'image_files.txt')) + lake.file.write_csv(f_lines, os.path.join(root_dir, "image_files.txt")) return f_lines def image_to_tfexample(image_data, image_format, height, width, class_id): - return tf.train.Example(features=tf.train.Features( - feature={'image/encoded': bytes_feature(image_data), - 'image/format': bytes_feature(image_format), - 'image/class/label': int64_feature(class_id), - 'image/height': int64_feature(height), - 'image/width': int64_feature(width), })) + return RecordWriter.image_to_tfexample( + image_data, image_format, height, width, class_id + ) def convert_to_tfrecord(f_lines, root_dir): - """Converts a file to TFRecords.""" - print('Generating TFRecords' ) - with tf.Graph().as_default(): - image_reader = ImageReader() - with tf.Session('') as sess: - piece_count = 15000 - num_pieces = int(len(f_lines) / piece_count + 1) - - for num_piece in range(num_pieces): - start_id, end_id = num_piece*piece_count, min(len(f_lines), (num_piece+1)*piece_count) - - output_file = os.path.join(root_dir, 'flowers_train_liby_' + str(num_piece+1).zfill(6) + '.tfrecord') - with tf.python_io.TFRecordWriter(output_file) as record_writer: - for index in range(start_id, end_id): - try: - cvmat = cv2.imread(f_lines[index][0]) - h, w, _ = cvmat.shape - except Exception as e: - print(e) - print("error image file {0}...".format(f_lines[index][0])) - continue - pdmat = noise_padd(cvmat, edge_size=max(h, w)) - encoded_image = cv2.imencode('.jpg', pdmat)[1].tostring() - - # encoded_image = tf.gfile.FastGFile(f_lines[index][0], 'rb').read() - # print("train image label: {0}".format(map_dict[cls_name])) - height, width = image_reader.read_image_dims(sess, encoded_image) - # tf exxmple format: NCHW - example = image_to_tfexample(encoded_image, b'jpg', height, width, f_lines[index][1]) - record_writer.write(example.SerializeToString()) - - print("finished: ", index + 1, '/', len(f_lines), "; image height: {0}, width: {1}".format(height, width)) - - sys.stdout.write('\n') - sys.stdout.flush() - - -if __name__ == '__main__': - # root_dir = '/data/dataset/liby_offline/train_renet50/train_datas' - # all_files_c = get_image_pairs(root_dir) - - train_image_file = '/data/dataset/liby_offline/train_renet50/models/train.txt' - with open(train_image_file, 'r') as f: - all_files = [item.strip().split(' ')[0] for item in f.readlines()] - - dir_dict = get_files_dir_dict(all_files) - label_map = get_label_dict(dir_dict) - - check_0 = all_files[0] - # shuffle all the files - random.shuffle(all_files) - assert check_0 != all_files[0] - - svae_dir = '/data/dataset/liby_offline/train_renet50/save_tf_records' - f_lines = write_laebl_lines(svae_dir, all_files, label_map) - - convert_to_tfrecord(f_lines, svae_dir) - + return RecordWriter.convert_to_tfrecord( + f_lines, root_dir, dataset_type="flowers_train_liby_" + ) diff --git a/imagedt/decorator/__init__.py b/imagedt/decorator/__init__.py index 4d46cdd..15791af 100644 --- a/imagedt/decorator/__init__.py +++ b/imagedt/decorator/__init__.py @@ -1,5 +1,14 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Compatibility exports, loaded on first use.""" -from .decorator_time import * \ No newline at end of file +from imagedt._lazy import resolve as _resolve + +_EXPORTS = {"time_cost": (".decorator_time", "time_cost")} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/decorator/decorator_time.py b/imagedt/decorator/decorator_time.py index 852deba..6e46ffd 100644 --- a/imagedt/decorator/decorator_time.py +++ b/imagedt/decorator/decorator_time.py @@ -1,14 +1,15 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import time +from functools import wraps def time_cost(func): + @wraps(func) def _time_cost(*args, **kwargs): - start = time.time() + start = time.perf_counter() ret = func(*args, **kwargs) - print("==> time-cost: %6f (s) %s " % (time.time() - start, func.__name__)) + print( + f"==> time-cost: {time.perf_counter() - start:6f} (s) {func.__name__} " + ) return ret - return _time_cost \ No newline at end of file + + return _time_cost diff --git a/imagedt/dir/__init__.py b/imagedt/dir/__init__.py index ae1c100..c7d6459 100644 --- a/imagedt/dir/__init__.py +++ b/imagedt/dir/__init__.py @@ -1,5 +1,18 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Compatibility exports, loaded on first use.""" -from .dir_loop import * \ No newline at end of file +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "LoopFile": (".dir_loop", "LoopFile"), + "loop": (".dir_loop", "loop"), + "get_dir_images": (".dir_loop", "get_dir_images"), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/dir/dir_loop.py b/imagedt/dir/dir_loop.py index 909e91b..4d646aa 100644 --- a/imagedt/dir/dir_loop.py +++ b/imagedt/dir/dir_loop.py @@ -1,59 +1,50 @@ -# coding: utf-8 +"""Deterministic directory traversal preserving the original callbacks.""" + import os import re class LoopFile: - def __init__(self, root_dir, file_extend=[], short_exclude=[], long_exclude=[]): - self.root_dir = root_dir - self.short_exclude = short_exclude - self.long_exclude = long_exclude - self.file_extend = file_extend - - def __del__(self): - pass - + def __init__( + self, root_dir, file_extend=None, short_exclude=None, long_exclude=None + ): + self.root_dir = os.fspath(root_dir) + self.file_extend = tuple(file_extend or ()) + self.short_exclude = tuple(short_exclude or ()) + self.long_exclude = tuple(long_exclude or ()) + self._patterns = [re.compile(p) for p in self.short_exclude] + def start(self, func): self.func = func return self.loop_file(self.root_dir) - + def loop_file(self, root_dir): - t_sum = [] - sub_gen = os.listdir(root_dir) - for sub in sub_gen: - is_exclude = False - for extends in self.short_exclude: ##在不检查文件、目录范围中 - if extends in sub: ##包含特定内容 - is_exclude = True - break - if re.search(extends, sub): ##匹配指定正则 - is_exclude = True - break - if is_exclude: - continue - abs_path = os.path.join(root_dir, sub) - is_exclude = False - for exclude in self.long_exclude: - if exclude == abs_path[-len(exclude):]: - is_exclude = True - break - if is_exclude: - continue - if os.path.isdir(abs_path): - t_sum.extend(self.loop_file(abs_path)) - elif os.path.isfile(abs_path): - if len(self.file_extend) > 0: - if not "." + abs_path.rsplit(".", 1)[1] in self.file_extend: ##不在后缀名 检查范围中 + results = [] + with os.scandir(root_dir) as entries: + for entry in sorted(entries, key=lambda e: e.name): + if any( + p in entry.name or r.search(entry.name) + for p, r in zip(self.short_exclude, self._patterns, strict=True) + ): + continue + if any(entry.path.endswith(p) for p in self.long_exclude): + continue + # Directory symlinks are excluded to prevent traversal cycles. + if entry.is_dir(follow_symlinks=False): + results.extend(self.loop_file(entry.path)) + elif entry.is_file(): + if ( + self.file_extend + and os.path.splitext(entry.name)[1] not in self.file_extend + ): continue - t_sum.append(self.func(abs_path)) - return t_sum + results.append(self.func(entry.path)) + return results -def loop(root, extensions=['.jpg', '.png']): - lf = LoopFile(root, extensions) - return lf.start(lambda f: f) +def loop(root, extensions=(".jpg", ".png")): + return LoopFile(root, extensions).start(lambda path: path) -def get_dir_images(root='./', extensions=['.jpg', '.png']): - lf = LoopFile(root, extensions) - return len(lf.start(lambda f: f)) \ No newline at end of file +def get_dir_images(root="./", extensions=(".jpg", ".png")): + return len(loop(root, extensions)) diff --git a/imagedt/file/__init__.py b/imagedt/file/__init__.py index 8ea8c05..48d1113 100644 --- a/imagedt/file/__init__.py +++ b/imagedt/file/__init__.py @@ -1,7 +1,26 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Compatibility exports, loaded on first use.""" -from .file_operate import * -from .parse_annotation import * -from .file_write import * +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "DIR_TYPR": (".file_operate", "DIR_TYPR"), + "FilesOp": (".file_operate", "FilesOp"), + "Anno_OP": (".parse_annotation", "Anno_OP"), + "write_txt": (".file_write", "write_txt"), + "write_csv": (".file_write", "write_csv"), + "read_csv": (".file_write", "read_csv"), + "readlines": (".file_write", "readlines"), + "write_json": (".file_write", "write_json"), + "read_json": (".file_write", "read_json"), + "decode_strs": (".file_write", "decode_strs"), + "writeExcel": (".file_write", "writeExcel"), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/file/file_operate.py b/imagedt/file/file_operate.py index 164e8d7..b3239bd 100644 --- a/imagedt/file/file_operate.py +++ b/imagedt/file/file_operate.py @@ -1,120 +1,130 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import os -import cv2 import uuid +import cv2 + from ..dir.dir_loop import loop -DIR_TYPR = ['JPEGImages', 'Annotations'] +DIR_TYPR = ["JPEGImages", "Annotations"] -class FilesOp(object): +class FilesOp: def __init__(self): - super(FilesOp, self).__init__() + super().__init__() def get_abs_path(self, file_dir, files_list): return [os.path.join(file_dir, f_item) for f_item in files_list] def check_dir_exist(self, path): if not os.path.isdir(path): - os.mkdir(path) - print("mkidr {0}".format(path)) + os.makedirs(path, exist_ok=True) + print(f"mkidr {path}") def check_file_exist(self, fpath): if not os.path.exists(fpath): - raise IOError("{0} not exist".format(fpath)) + raise OSError(f"{fpath} not exist") def fillter_dirname(self, file_paths): lines = list() for file_path in file_paths: - imtes = file_path.strip().split(',') + imtes = file_path.strip().split(",") bsname = os.path.basename(imtes[0]) lines.append([bsname, imtes[1]]) return lines def fillter_file_type(self, file_paths): - return [''.join(file_path.split('.')[0]) for file_path in file_paths] + return [os.path.splitext(file_path)[0] for file_path in file_paths] def rename_with_uuid(self, fname, uid): dir_name = os.path.dirname(fname) bs_name = os.path.basename(fname) try: - self.assert_in_op('.', bs_name) - except Exception as e: - print(e) - raise ValueError("file {0} basename error ".format(bs_name)) - return os.path.join(dir_name, uid+'.'+bs_name.split('.')[-1]) + self.assert_in_op(".", bs_name) + except ValueError as e: + print(e) + raise ValueError(f"file {bs_name} basename error ") from e + return os.path.join(dir_name, uid + "." + bs_name.split(".")[-1]) def rename_data_pairs(self, file_pair, uid): - for f_path in file_pair: - new_fpath = self.rename_with_uuid(f_path, uid) - os.rename(f_path, new_fpath) + changes = [(path, self.rename_with_uuid(path, uid)) for path in file_pair] + for path, destination in changes: + self.check_file_exist(path) + if path != destination and os.path.exists(destination): + raise FileExistsError(destination) + completed = [] + try: + for path, destination in changes: + os.rename(path, destination) + completed.append((path, destination)) + except OSError: + for path, destination in reversed(completed): + os.rename(destination, path) + raise def assert_not_in_op(self, member, container): - try: - assert member not in container - except Exception as e: - print(e) - raise ValueError('Value {0} contains {1}'.format(container, member)) + if member in container: + raise ValueError(f"Value {container} contains {member}") def assert_in_op(self, member, container): - try: - if isinstance(member, type(list())): - for assert_str in member: - assert assert_str in container - else: - assert member in container - except Exception as e: - print(e) - raise ValueError("Value {0} didn't contain {1}".format(container, member)) - - def assert_instance_op(self, inst, tar_inst=list()): - try: - assert type(inst) == type(tar_inst) - except Exception as e: - print(e) - raise ValueError("input Values should be list...") + members = member if isinstance(member, list) else [member] + if any(item not in container for item in members): + raise ValueError(f"Value {container} didn't contain {member}") + + def assert_instance_op(self, inst, tar_inst=None): + expected = list if tar_inst is None else type(tar_inst) + if type(inst) is not expected: + raise ValueError(f"input Values should be {expected.__name__}") def del_broken_image(self, root_dir): image_files = os.listdir(os.path.join(root_dir, DIR_TYPR[0])) for index, img in enumerate(image_files): - img_name = os.path.join(root_dir, 'JPEGImages', img) + img_name = os.path.join(root_dir, "JPEGImages", img) imat = cv2.imread(img_name) if imat is None: os.remove(img_name) - print("remove broken image: {0}".format(img)) - print("check_data_pairs finished: {0} / {1}".format(index + 1, len(image_files))) + print(f"remove broken image: {img}") + print(f"check_data_pairs finished: {index + 1} / {len(image_files)}") - def check_data_pairs(self, root_dir, anno_type='.xml'): + def check_data_pairs(self, root_dir, anno_type=".xml"): # root dir should contain 'JPEGImages' and 'Annotations' dirs self.assert_in_op(DIR_TYPR, os.listdir(root_dir)) - images_files = self.fillter_file_type(os.listdir(os.path.join(root_dir, DIR_TYPR[0]))) - allxml_files = self.fillter_file_type(os.listdir(os.path.join(root_dir, DIR_TYPR[1]))) + images_files = self.fillter_file_type( + os.listdir(os.path.join(root_dir, DIR_TYPR[0])) + ) + allxml_files = self.fillter_file_type( + os.listdir(os.path.join(root_dir, DIR_TYPR[1])) + ) + image_names = set(images_files) num_images = len(images_files) num_xmlfil = len(allxml_files) - if num_images == num_xmlfil and len(set(images_files+allxml_files)) == num_images: - print("check_data_pairs finished: effective data pairs {0} . ".format(len(images_files))) + if ( + num_images == num_xmlfil + and len(set(images_files + allxml_files)) == num_images + ): + print( + f"check_data_pairs finished: effective data pairs {len(images_files)} . " + ) return "have same files" for index, file_name in enumerate(allxml_files): - if file_name not in images_files: - print("annotation file {0} not in JPG".format(file_name)) - os.remove(os.path.join(root_dir, DIR_TYPR[1], file_name+anno_type)) - print("check_data_pairs finished: {0} / {1}".format(index+1, num_xmlfil)) + if file_name not in image_names: + print(f"annotation file {file_name} not in JPG") + os.remove(os.path.join(root_dir, DIR_TYPR[1], file_name + anno_type)) + print(f"check_data_pairs finished: {index + 1} / {num_xmlfil}") - def get_jpg_xml_pairs(self, root_dir, anno_type='.xml'): + def get_jpg_xml_pairs(self, root_dir, anno_type=".xml"): """return: image and xml pairs abs path [image_path, xml_path]......""" file_pairs = list() allxml_dir = os.path.join(root_dir, DIR_TYPR[1]) images_dir = os.path.join(root_dir, DIR_TYPR[0]) - xml_files = self.get_abs_path(allxml_dir, os.listdir(allxml_dir)) + xml_files = self.get_abs_path( + allxml_dir, + sorted(name for name in os.listdir(allxml_dir) if name.endswith(anno_type)), + ) for xml_path in xml_files: - imgae_bsname = os.path.basename(xml_path).replace(anno_type, '.jpg') + imgae_bsname = os.path.splitext(os.path.basename(xml_path))[0] + ".jpg" image_path = os.path.join(images_dir, imgae_bsname) file_pairs.append([image_path, xml_path]) return file_pairs @@ -128,7 +138,7 @@ def rename_file_with_uuid(self, root_dir): for index, file_pair in enumerate(file_pairs): uid = str(uuid.uuid1()) self.rename_data_pairs(file_pair, uid) - print("finished: {0} / {1}.".format(index+1, len(file_pairs))) + print(f"finished: {index + 1} / {len(file_pairs)}.") def rename_single_file_uuid(self, root_dir): f_names = self.get_abs_path(root_dir, os.listdir(root_dir)) @@ -136,28 +146,28 @@ def rename_single_file_uuid(self, root_dir): f_uuid = str(uuid.uuid1()) new_fpath = self.rename_with_uuid(f_name, f_uuid) os.rename(f_name, new_fpath) - print("finished {0}/{1}".format(index+1, len(f_names))) + print(f"finished {index + 1}/{len(f_names)}") def loop_dir(self, root_dir): dep_files = list() dirs = self.get_abs_path(root_dir, os.listdir(root_dir)) for item in dirs: + if not os.path.isdir(item): + continue dep_files += self.get_abs_path(item, os.listdir(item)) return dep_files - def rename_class_dir(self, root_dir, before_cls_str='other'): - all_files = loop(root_dir, ['.jpg']) - all_dirs = dict([(item, 1) for item in all_files]) - - for class_dir in all_dirs: - class_dir_name = os.path.dirname(class_dir) - set_name = class_dir_name.replace(os.path.basename(class_dir_name), before_cls_str + os.path.basename(class_dir_name)) + def rename_class_dir(self, root_dir, before_cls_str="other"): + directories = {os.path.dirname(path) for path in loop(root_dir, [".jpg"])} + for path in sorted(directories, key=lambda p: p.count(os.sep), reverse=True): + basename = os.path.basename(path) + if not basename.startswith(before_cls_str): + destination = os.path.join( + os.path.dirname(path), before_cls_str + basename + ) + if os.path.exists(destination): + raise FileExistsError(destination) + os.rename(path, destination) - if not class_dir.startswith(before_cls_str): - try: - os.renames(class_dir_name, set_name) - print("rename {0} as {1}".format(os.path.basename(class_dir_name), os.path.basename(set_name))) - except Exception as e: - print(class_dir_name) -FilesOp = FilesOp() \ No newline at end of file +FilesOp = FilesOp() diff --git a/imagedt/file/file_write.py b/imagedt/file/file_write.py index 206ceb3..02c55fe 100644 --- a/imagedt/file/file_write.py +++ b/imagedt/file/file_write.py @@ -1,74 +1,67 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import csv import json -import xlwt -import sys -def write_txt(data, path, separator=','): - with open(path, "w") as text_file: - if isinstance(data, (str, unicode)): +def write_txt(data, path, separator=","): + with open(path, "w", encoding="utf-8") as text_file: + if isinstance(data, str): text_file.write(str(data)) elif isinstance(data, list): for line in data: - if isinstance(line, (str, unicode)): - text_file.write(str(line) + '\n') + if isinstance(line, str): + text_file.write(str(line) + "\n") elif isinstance(line, list): - line = map(str, line) - split_str = separator+'' + line = list(map(str, line)) + split_str = separator + "" line = split_str.join(line) - text_file.write(str(line) + '\n') + text_file.write(str(line) + "\n") else: text_file.write(json.dumps(data)) def write_csv(data, path, is_excel=False): - with open(path, 'w') as csvfile: - if is_excel: - csvfile.write(u"\ufeff") - writer = csv.writer(csvfile) - for row in data: - writer.writerow(row) + with open(path, "w", encoding="utf-8", newline="") as csvfile: + if is_excel: + csvfile.write("\ufeff") + writer = csv.writer(csvfile) + for row in data: + writer.writerow(row) -def read_csv(path, delimiter=','): - with open(path, 'r') as csvfile: +def read_csv(path, delimiter=","): + with open(path, encoding="utf-8-sig", newline="") as csvfile: reader = csv.reader(csvfile, delimiter=delimiter) data = [line for line in reader] return data def readlines(file_path): - with open(file_path, 'r') as f: + with open(file_path, encoding="utf-8") as f: lines = [item.strip() for item in f.readlines()] return lines def write_json(json_objects, file_path): - with open(file_path, 'w') as f: + with open(file_path, "w", encoding="utf-8") as f: json.dump(json_objects, f) def read_json(file_path): - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: items = json.load(f) return items def decode_strs(strs): - # py2 - if sys.hexversion < 0x3000000: - strs = strs.decode('utf-8') - return strs + return strs.decode("utf-8") if isinstance(strs, bytes) else strs def writeExcel(write_infos_mul, excel_path, sheetNames=None, set_color=False): + import xlwt + workbook = xlwt.Workbook() # style = xlwt.easyxf('align: wrap on') - style1 = xlwt.easyxf('pattern: pattern solid, fore_colour red;') + xlwt.easyxf("pattern: pattern solid, fore_colour red;") for ind, write_infos in enumerate(write_infos_mul): if sheetNames is None: sheetname = str(ind) @@ -77,9 +70,8 @@ def writeExcel(write_infos_mul, excel_path, sheetNames=None, set_color=False): sheetname = decode_strs(sheetname) writeSheet = workbook.add_sheet(sheetname) for row, item_infos in enumerate(write_infos): - unnormal_shop = item_infos[-2] for idx, info in enumerate(item_infos): - if type(info) == type('sss'): + if isinstance(info, str): info = decode_strs(info) writeSheet.write(row, idx, info) workbook.save(excel_path) diff --git a/imagedt/file/parse_annotation.py b/imagedt/file/parse_annotation.py index ac7ae16..2eae495 100644 --- a/imagedt/file/parse_annotation.py +++ b/imagedt/file/parse_annotation.py @@ -1,97 +1,128 @@ -# coding: utf-8 -import os import csv import math +import os + import numpy as np from lxml import etree -class Anno_OP(object): +class Anno_OP: def __init__(self): - super(Anno_OP, self).__init__() + super().__init__() def read_xml(self, fxml): """ :param fxml: xml file path. :return: annotation is object: annotation.iter('object'). """ - annotation = etree.parse(fxml).getroot() + annotation = etree.parse( + fxml, etree.XMLParser(resolve_entities=False, no_network=True) + ).getroot() return annotation def parse_bndx_object(self, bndbox_object): """return bndx list ['xmin', 'ymin', 'xmax', 'ymax'] by float""" - rectangle = [bndbox_object.find(coord).text for coord in ['xmin', 'ymin', 'xmax', 'ymax']] + rectangle = [ + bndbox_object.find(coord).text for coord in ["xmin", "ymin", "xmax", "ymax"] + ] return list(map(float, rectangle)) def read_txt_bndbox(self, fpath): - with open(fpath, 'r') as f: + with open(fpath) as f: lines = f.readlines() - return [line.strip().split(',') for line in lines] + return [line.strip().split(",") for line in lines] def get_object_bndbox(self, sku_object): - return map(float, [sku_object['bndbox'][ite_cor] for ite_cor in ['xmin', 'ymin', 'xmax', 'ymax']]) + return list( + map( + float, + [ + sku_object["bndbox"][ite_cor] + for ite_cor in ["xmin", "ymin", "xmax", "ymax"] + ], + ) + ) def get_label_object(self, fxml): objects = [] annotation = self.read_xml(fxml) - for iter_object in annotation.iter('object'): - bndx = self.parse_bndx_object(iter_object.find('bndbox')) - object_find = {'name': iter_object.find('name').text, - 'bndbox':{'xmin':bndx[0], 'ymin':bndx[1], 'xmax':bndx[2], 'ymax':bndx[3]}} - if iter_object.find('price') is not None: - object_find['price'] = iter_object.find('price').text - if iter_object.find('classify_confidence') is not None: - object_find['classify_confidence'] = iter_object.find('classify_confidence').text - if iter_object.find('desc') is not None: - object_find['desc'] = iter_object.find('desc').text - if iter_object.find('prices') is not None: - object_find['prices'] = [] - for price_obj in iter_object.iter('prices'): - object_find['prices'].append(price_obj.find('price').text) - if iter_object.find('blur') is not None: - object_find['blur'] = iter_object.find('blur').text + for iter_object in annotation.iter("object"): + bndx = self.parse_bndx_object(iter_object.find("bndbox")) + object_find = { + "name": iter_object.find("name").text, + "bndbox": { + "xmin": bndx[0], + "ymin": bndx[1], + "xmax": bndx[2], + "ymax": bndx[3], + }, + } + if iter_object.find("price") is not None: + object_find["price"] = iter_object.find("price").text + if iter_object.find("classify_confidence") is not None: + object_find["classify_confidence"] = iter_object.find( + "classify_confidence" + ).text + if iter_object.find("desc") is not None: + object_find["desc"] = iter_object.find("desc").text + if iter_object.find("prices") is not None: + object_find["prices"] = [] + for price_obj in iter_object.iter("prices"): + object_find["prices"].append(price_obj.find("price").text) + if iter_object.find("blur") is not None: + object_find["blur"] = iter_object.find("blur").text objects.append(object_find) return objects def parse_lab_xml(self, fxml): annotation = self.read_xml(fxml) - return [self.parse_bndx_object(iter_object.find('bndbox')) for iter_object in annotation.iter('object')] + return [ + self.parse_bndx_object(iter_object.find("bndbox")) + for iter_object in annotation.iter("object") + ] def load_annoataion(self, poly_file): - ''' + """ load annotation from the text file :param p: :return: - ''' + """ text_polys = [] text_tags = [] if not os.path.exists(poly_file): - return np.array(text_polys, dtype=np.float32) - with open(poly_file, 'r') as f: + return np.empty((0, 4, 2), dtype=np.float32), np.empty(0, dtype=bool) + with open(poly_file, encoding="utf-8-sig", newline="") as f: reader = csv.reader(f) for line in reader: label = line[-1] # strip BOM. \ufeff for python3, \xef\xbb\bf for python2 - line = [i.strip('\ufeff').strip('\xef\xbb\xbf') for i in line] + line = [i.lstrip("\ufeff") for i in line] x1, y1, x2, y2, x3, y3, x4, y4 = list(map(float, line[:8])) text_polys.append([[x1, y1], [x2, y2], [x3, y3], [x4, y4]]) - if label == '*' or label == '###': + if label == "*" or label == "###": text_tags.append(True) else: text_tags.append(False) - return np.array(text_polys, dtype=np.float32), np.array(text_tags, dtype=np.bool) + return np.asarray(text_polys, dtype=np.float32).reshape( + -1, 4, 2 + ), np.asarray(text_tags, dtype=bool) def parse_poly_locations(self, loca_info): - xmin, ymin = map(math.floor, loca_info[0]) - xmax, ymax = map(math.ceil, loca_info[2]) + points = np.asarray(loca_info, dtype=float) + xmin, ymin = map(math.floor, points.min(axis=0)) + xmax, ymax = map(math.ceil, points.max(axis=0)) return xmin, ymin, xmax, ymax def convert_poly_to_Xml(self, poly_file): loca_infos, has_fonts = self.load_annoataion(poly_file) - xml_dir = os.path.join(os.path.dirname(os.path.dirname(poly_file)), 'Annotations_xml') - xml_name = os.path.join(xml_dir, os.path.basename(poly_file).replace('.txt', '.xml')) + xml_dir = os.path.join( + os.path.dirname(os.path.dirname(poly_file)), "Annotations_xml" + ) + xml_name = os.path.join( + xml_dir, os.path.basename(poly_file).replace(".txt", ".xml") + ) if not os.path.exists(xml_dir): os.mkdir(xml_dir) @@ -101,15 +132,17 @@ def convert_poly_to_Xml(self, poly_file): vocObject = etree.SubElement(annotation, "object") name = etree.SubElement(vocObject, "name") - if has_fonts[index] == True: + if has_fonts[index]: # fonts are blur - name.text = '2' + name.text = "2" else: - name.text = '1' + name.text = "1" desc = etree.SubElement(vocObject, "desc") - desc.text = unicode('文字文本', 'utf-8') - xmins, ymins, xmaxs, ymaxs = map(str, self.parse_poly_locations(loca_info)) + desc.text = "文字文本" + xmins, ymins, xmaxs, ymaxs = list( + map(str, self.parse_poly_locations(loca_info)) + ) bndbox = etree.SubElement(vocObject, "bndbox") xmin = etree.SubElement(bndbox, "xmin") @@ -121,84 +154,83 @@ def convert_poly_to_Xml(self, poly_file): ymax = etree.SubElement(bndbox, "ymax") ymax.text = ymaxs - xml = etree.tostring(annotation, pretty_print=True, encoding='UTF-8') - with open(xml_name, "w") as xmlfile: + xml = etree.tostring(annotation, pretty_print=True, encoding="UTF-8") + with open(xml_name, "wb") as xmlfile: xmlfile.write(xml) - def reset_xml_cls(self, xml_dir, name='3488', desc=u'圆柱圆台形',resetid_list=None ): - if resetid_list is not None: - try: - assert type(resetid_list) == type([]) - except: - raise ValueError('resetid_list must be list: like [3477,3478,12354]') + def reset_xml_cls(self, xml_dir, name="3488", desc="圆柱圆台形", resetid_list=None): + if resetid_list is not None and not isinstance(resetid_list, list): + raise ValueError("resetid_list must be list: like [3477,3478,12354]") for index, f_path in enumerate(os.listdir(xml_dir)): xml_path = os.path.join(xml_dir, f_path) an_objects = self.read_xml(xml_path) - for tag in an_objects.iter('object'): + for tag in list(an_objects.iter("object")): if resetid_list is not None: - if int(tag.find('name').text) in resetid_list: - tag.find('name').text = name + if int(tag.find("name").text) in resetid_list: + tag.find("name").text = str(name) else: an_objects.remove(tag) else: - tag.find('name').text = name - if tag.find('desc') is not None: - tag.find('desc').text = desc - xml = etree.tostring(an_objects, pretty_print=True, encoding='UTF-8') - with open(xml_path, "w") as xmlfile: + tag.find("name").text = str(name) + if tag.find("desc") is not None: + tag.find("desc").text = desc + xml = etree.tostring(an_objects, pretty_print=True, encoding="UTF-8") + with open(xml_path, "wb") as xmlfile: xmlfile.write(xml) - print("finished reset {0}/{1}".format(index+1, len(os.listdir(xml_dir)))) - + print(f"finished reset {index + 1}/{len(os.listdir(xml_dir))}") - def filter_xml_classes(self, xml_dir, name='0', desc=u'other', resetid_list=None): - if resetid_list is not None: - try: - assert type(resetid_list) == type([]) - except: - raise ValueError('resetid_list must be list: like [3477,3478,12354]') + def filter_xml_classes(self, xml_dir, name="0", desc="other", resetid_list=None): + if resetid_list is not None and not isinstance(resetid_list, list): + raise ValueError("resetid_list must be list: like [3477,3478,12354]") for index, f_path in enumerate(os.listdir(xml_dir)): xml_path = os.path.join(xml_dir, f_path) - print (xml_path) + print(xml_path) an_objects = self.read_xml(xml_path) - for tag in an_objects.iter('object'): - if int(tag.find('name').text) not in resetid_list: - tag.find('name').text = name - - xml = etree.tostring(an_objects, pretty_print=True, encoding='UTF-8') - with open(xml_path, "w") as xmlfile: + for tag in list(an_objects.iter("object")): + if ( + resetid_list is None + or int(tag.find("name").text) not in resetid_list + ): + tag.find("name").text = str(name) + + xml = etree.tostring(an_objects, pretty_print=True, encoding="UTF-8") + with open(xml_path, "wb") as xmlfile: xmlfile.write(xml) - print("finished reset {0}/{1}".format(index+1, len(os.listdir(xml_dir)))) - + print(f"finished reset {index + 1}/{len(os.listdir(xml_dir))}") def write_anno_object(self, annotation, xmlname): - xml = etree.tostring(annotation, pretty_print=True, encoding='UTF-8') - with open(xmlname, "w") as xmlfile: + xml = etree.tostring(annotation, pretty_print=True, encoding="UTF-8") + with open(xmlname, "wb") as xmlfile: xmlfile.write(xml) - def wirte_xml_file(self, xml_etree, xml_path): - with open(xml_path, "w") as xmlfile: - xmlfile.write(xml_etree) - + with open(xml_path, "wb") as xmlfile: + xmlfile.write( + xml_etree.encode("utf-8") if isinstance(xml_etree, str) else xml_etree + ) def write_xml(self, xmlname, bndboxs, scores, classes, thresh=0): + if len(bndboxs) != len(scores) or ( + not isinstance(classes, str) and len(classes) != len(bndboxs) + ): + raise ValueError("boxes, scores and classes must have equal length") annotation = etree.Element("Annotation") for index, loca_info in enumerate(bndboxs): if scores[index] < thresh: continue vocObject = etree.SubElement(annotation, "object") name = etree.SubElement(vocObject, "name") - name.text = classes[index] + name.text = str(classes if isinstance(classes, str) else classes[index]) desc = etree.SubElement(vocObject, "desc") - desc.text = unicode('object', 'utf-8') + desc.text = "object" confi = etree.SubElement(vocObject, "confidence") confi.text = str(scores[index]) # xmin ,ymin, xmax, ymax - xmins, ymins, xmaxs, ymaxs = map(str, loca_info) + xmins, ymins, xmaxs, ymaxs = list(map(str, loca_info)) # ymin, xmin, ymax, xmax # ymins, xmins, ymaxs, xmaxs = map(str, loca_info) @@ -214,7 +246,8 @@ def write_xml(self, xmlname, bndboxs, scores, classes, thresh=0): self.write_anno_object(annotation, xmlname) # xml = etree.tostring(annotation, pretty_print=True, encoding='UTF-8') - # with open(xmlname, "w") as xmlfile: + # with open(xmlname, "wb") as xmlfile: # xmlfile.write(xml) -Anno_OP = Anno_OP() \ No newline at end of file + +Anno_OP = Anno_OP() diff --git a/imagedt/image/__init__.py b/imagedt/image/__init__.py index 4b55959..f6ba2b0 100644 --- a/imagedt/image/__init__.py +++ b/imagedt/image/__init__.py @@ -1,9 +1,39 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Compatibility exports, loaded on first use.""" -from .image_pil import * -from .process import * +from imagedt._lazy import resolve as _resolve +_EXPORTS = { + "alpha_to_color": (".image_pil", "alpha_to_color"), + "IMG_EXTENSIONS": (".process", "IMG_EXTENSIONS"), + "clear_dir": (".process", "clear_dir"), + "noise_padd": (".process", "noise_padd"), + "remove_broken_image": (".process", "remove_broken_image"), + "resize_with_scale": (".process", "resize_with_scale"), + "padd_pixel": (".process", "padd_pixel"), + "reduce_imagenet_mean": (".process", "reduce_imagenet_mean"), + "swap_chanel_to_RGB": (".process", "swap_chanel_to_RGB"), + "vgg_preprocesing": (".process", "vgg_preprocesing"), + "inception_preprocesing": (".process", "inception_preprocesing"), + "our_processing": (".process", "our_processing"), + "md5": (".process", "md5"), + "get_box": (".process", "get_box"), + "put_cvtext": (".process", "put_cvtext"), + "draw_cvrect": (".process", "draw_cvrect"), + "cv_show": (".process", "cv_show"), + "resize_dir_images_with_scale": (".process", "resize_dir_images_with_scale"), + "points_to_contour": (".process", "points_to_contour"), + "points_to_contours": (".process", "points_to_contours"), + "put_text": (".process", "put_text"), + "draw_contours": (".process", "draw_contours"), + "polly_points2contours": (".process", "polly_points2contours"), + "rename_imfiles_with_md5": (".process", "rename_imfiles_with_md5"), +} +__all__ = list(_EXPORTS) -IMG_EXTENSIONS = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP'] + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/image/image_pil.py b/imagedt/image/image_pil.py index 0c280a1..a329977 100644 --- a/imagedt/image/image_pil.py +++ b/imagedt/image/image_pil.py @@ -1,7 +1,5 @@ -# coding: utf-8 -from PIL import Image import numpy as np -import collections +from PIL import Image def alpha_to_color(image, color=(255, 255, 255)): @@ -12,7 +10,7 @@ def alpha_to_color(image, color=(255, 255, 255)): Keyword Arguments: image -- PIL RGBA Image object color -- Tuple r, g, b (default 255, 255, 255) - """ + """ x = np.array(image) r, g, b, a = np.rollaxis(x, axis=-1) # print np.where(a != 0) @@ -21,4 +19,4 @@ def alpha_to_color(image, color=(255, 255, 255)): b[a == 0] = color[2] a = np.ones_like(a) * 255 x = np.dstack([r, g, b, a]) - return Image.fromarray(x, 'RGBA') + return Image.fromarray(x, "RGBA") diff --git a/imagedt/image/process.py b/imagedt/image/process.py index 9addb55..7fc9e59 100644 --- a/imagedt/image/process.py +++ b/imagedt/image/process.py @@ -1,19 +1,24 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - +import hashlib import os + import cv2 -import sys -import hashlib import numpy as np -from multiprocessing import Process # from PIL import Image - from ..dir.dir_loop import loop -IMG_EXTENSIONS = ['.jpg', '.JPG', '.jpeg', '.JPEG', '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP'] +IMG_EXTENSIONS = [ + ".jpg", + ".JPG", + ".jpeg", + ".JPEG", + ".png", + ".PNG", + ".ppm", + ".PPM", + ".bmp", + ".BMP", +] _R_MEAN = 123.68 _G_MEAN = 116.78 @@ -21,73 +26,50 @@ def clear_dir(images_dir, debug=True, multi_process=4): - """清理目录下破损图片""" - images_path = loop(images_dir, IMG_EXTENSIONS) - count = len(images_path) - def delete_bro_img(image_lists): - for i, image_path in enumerate(image_lists, 1): - if debug: - sys.stdout.write('\r%d/%d' % (i, len(image_lists))) - sys.stdout.flush() - try: - cv2.imread(image_path).shape - # Image.open(image_path).convert('RGB') - except Exception as e: - print("Remove broken image: {0}".format(image_path)) - os.remove(image_path) - for i in range(multi_process): - start_ind = (count / multi_process)*i - end_ind = -1 if (i+1) == count else (count / multi_process)*(i+1) - image_lists = images_path[start_ind:end_ind] - p = Process(target=delete_bro_img, args=(image_lists,)) - p.start() - print("start job {0}, start_ind:end_inf --> {1}:{2} ......".format(i+1, start_ind, end_ind)) - # p.join() + """Remove unreadable images and wait for all workers to finish.""" + from concurrent.futures import ThreadPoolExecutor + if not isinstance(multi_process, int) or multi_process < 1: + raise ValueError("multi_process must be a positive integer") + paths = loop(images_dir, IMG_EXTENSIONS) + + def check(path): + if cv2.imread(path) is None: + os.remove(path) + if debug: + print(f"Remove broken image: {path}") + + with ThreadPoolExecutor(max_workers=multi_process) as executor: + list(executor.map(check, paths)) + + +def _pad_image(img, width, height, low, high): + if img.ndim != 3 or img.shape[2] != 3 or min(img.shape[:2]) < 1: + raise ValueError("expected a nonempty HWC image with three channels") + if ( + not isinstance(width, int) + or not isinstance(height, int) + or min(width, height) < 1 + ): + raise ValueError("target dimensions must be positive integers") + if not 0 <= low < high <= 256: + raise ValueError("pixel range must satisfy 0 <= low < high <= 256") + h, w = img.shape[:2] + ratio = min(width / w, height / h) + rw = min(width, max(1, round(w * ratio))) + rh = min(height, max(1, round(h * ratio))) + # Retain the old symmetric padding behavior, including parity adjustment. + rw = min(width, rw + (width - rw) % 2) + rh = min(height, rh + (height - rh) % 2) + resized = cv2.resize(img, (rw, rh), interpolation=cv2.INTER_LINEAR) + canvas = np.random.randint(low, high, (height, width, 3)).astype(np.uint8) + y, x = (height - rh) // 2, (width - rw) // 2 + canvas[y : y + rh, x : x + rw] = resized + return canvas def noise_padd(img, edge_size=224, start_pixel_value=0, end_pixel_value=256): - """ - img: cvMat - edge_size: image max edge - - return: cvMat [rectangle, height=width=edge_size] - """ - h, w, _ = img.shape - - width_ratio = float(w) / edge_size - height_ratio = float(h) / edge_size - if width_ratio > height_ratio: - resize_width = edge_size - resize_height = int(round(h / width_ratio)) - if (edge_size - resize_height) % 2 == 1: - resize_height += 1 - else: - resize_height = edge_size - resize_width = int(round(w / height_ratio)) - if (edge_size - resize_width) % 2 == 1: - resize_width += 1 - img = cv2.resize(img, (int(resize_width), int(resize_height)), interpolation=cv2.INTER_LINEAR) - - channels = 3 - # fill ends of dimension that is too short with random noise - if width_ratio > height_ratio: - padding = (edge_size - resize_height) / 2 - noise_size = (padding, edge_size) - if channels > 1: - noise_size += (channels,) - noise = np.random.randint(start_pixel_value, end_pixel_value, noise_size).astype('uint8') - # noise = np.zeros(noise_size).astype('uint8') - img = np.concatenate((noise, img, noise), axis=0) - else: - padding = (edge_size - resize_width) / 2 - noise_size = (edge_size, padding) - if channels > 1: - noise_size += (channels,) - noise = np.random.randint(start_pixel_value, end_pixel_value, noise_size).astype('uint8') - # noise = np.zeros(noise_size).astype('uint8') - img = np.concatenate((noise, img, noise), axis=1) - return img + return _pad_image(img, edge_size, edge_size, start_pixel_value, end_pixel_value) def remove_broken_image(data_dir): @@ -98,92 +80,54 @@ def remove_broken_image(data_dir): img_mat = cv2.imread(image_file) if img_mat is None: os.remove(image_file) - print('remove broken image {0}'.format(image_file)) - except: + print(f"remove broken image {image_file}") + except cv2.error: os.remove(image_file) - print('remove broken file {0}'.format(image_file)) + print(f"remove broken file {image_file}") def resize_with_scale(cvmat, max_length): - h, w, _ = cvmat.shape + if max_length <= 0: + raise ValueError("max_length must be positive") + h, w = cvmat.shape[:2] max_edge = max(h, w) if max_edge > max_length: ratio = float(max_edge) / max_length - width, height = w/ratio, h/ratio - cvmat = cv2.resize(cvmat, (int(width), int(height))) + width, height = w / ratio, h / ratio + cvmat = cv2.resize(cvmat, (max(1, int(width)), max(1, int(height)))) return cvmat def padd_pixel(img, edge_size_w=200, edge_size_h=96): - h, w, _ = img.shape - - width_ratio = float(w) / edge_size_w - if width_ratio > 1: - img = cv2.resize(img, (int(w/width_ratio), int(h/width_ratio))) - h, w, _ = img.shape - width_ratio = float(w) / edge_size_w - - height_ratio = float(h) / edge_size_h - if height_ratio > 1: - img = cv2.resize(img, (int(w/height_ratio), int(h/height_ratio))) - h, w, _ = img.shape - height_ratio = float(h) / edge_size_h - - if width_ratio > height_ratio: - resize_width = edge_size_w - resize_height = int(round(h / width_ratio)) - if (edge_size_h - resize_height) % 2 == 1: - resize_height += 1 - else: - resize_height = edge_size_h - resize_width = int(round(w / height_ratio)) - if (edge_size_w - resize_width) % 2 == 1: - resize_width += 1 - img = cv2.resize(img, (int(resize_width), int(resize_height)), interpolation=cv2.INTER_LINEAR) - - channels = 3 - if width_ratio > height_ratio: - padding = (edge_size_h - resize_height) / 2 - noise_size = (padding, edge_size_w) - if channels > 1: - noise_size += (channels,) - noise = np.random.randint(230, 240, noise_size).astype('uint8') - img = np.concatenate((noise, img, noise), axis=0) - else: - padding = (edge_size_w - resize_width) / 2 - noise_size = (edge_size_h, padding) - if channels > 1: - noise_size += (channels,) - noise = np.random.randint(230, 240, noise_size).astype('uint8') - img = np.concatenate((noise, img, noise), axis=1) - return img + return _pad_image(img, edge_size_w, edge_size_h, 230, 240) def reduce_imagenet_mean(cv_mat): - [B, G, R] = cv2.split(cv_mat) - return cv2.merge([B-_B_MEAN, G-_G_MEAN, R-_R_MEAN]) + [B, G, R] = cv2.split(cv_mat) + return cv2.merge([B - _B_MEAN, G - _G_MEAN, R - _R_MEAN]) def swap_chanel_to_RGB(cvmat): - return cv2.cvtColor(cvmat, cv2.COLOR_BGR2RGB) + return cv2.cvtColor(cvmat, cv2.COLOR_BGR2RGB) def vgg_preprocesing(cvmat): - cvmat = np.array(cvmat, dtype=np.float32) - cvmat -= [_R_MEAN,_G_MEAN,_B_MEAN] - return cvmat + cvmat = np.array(cvmat, dtype=np.float32) + cvmat -= [_R_MEAN, _G_MEAN, _B_MEAN] + return cvmat def inception_preprocesing(cvmat): - cvmat = (cvmat / 255. - 0.5) * 2 - cvmat = np.array(cvmat, dtype=np.float32) - return cvmat + cvmat = (cvmat / 255.0 - 0.5) * 2 + cvmat = np.array(cvmat, dtype=np.float32) + return cvmat + +def our_processing(cvmat): + cvmat = reduce_imagenet_mean(cvmat) + cvmat = np.array(cvmat, dtype=np.float32) + return np.expand_dims(cvmat, axis=0) -def our_processing(cvmat): - cvmat = reduce_imagenet_mean(cvmat) - cvmat = np.array(cvmat, dtype=np.float32) - return np.expand_dims(cvmat, axis=0) def md5(fname): hash_md5 = hashlib.md5() @@ -193,37 +137,58 @@ def md5(fname): return hash_md5.hexdigest() -def get_box(bndbox, loca_type='mid'): - if loca_type == 'top': +def get_box(bndbox, loca_type="mid"): + if loca_type == "top": # xmid = bndbox[0]+(bndbox[2]-bndbox[0])/2. xmid = bndbox[0] ymid = bndbox[1] - elif loca_type == 'bot': + elif loca_type == "bot": xmid = bndbox[0] - ymid = max(0, bndbox[3]-5) - elif loca_type == 'botup': + ymid = max(0, bndbox[3] - 5) + elif loca_type == "botup": xmid = bndbox[0] - ymid = max(0, bndbox[3]-(bndbox[3]-bndbox[1])/4) + ymid = max(0, bndbox[3] - (bndbox[3] - bndbox[1]) / 4) else: # xmid = bndbox[0]+(bndbox[2]-bndbox[0])/2. xmid = bndbox[0] - ymid = bndbox[1]+(bndbox[3]-bndbox[1])/2. + ymid = bndbox[1] + (bndbox[3] - bndbox[1]) / 2.0 return (int(xmid), int(ymid)) -def put_cvtext(img, text, posi, color=(0,0,255), fontScale=1, thickness=1, font_type=None): +def put_cvtext( + img, text, posi, color=(0, 0, 255), fontScale=1, thickness=1, font_type=None +): if font_type is None: font = cv2.FONT_HERSHEY_DUPLEX else: font = font_type - cv2.putText(img=img, text=text, org=posi, fontFace=font, fontScale=fontScale, - color=color, thickness=thickness, lineType=2) - - -def draw_cvrect(img, bndboxes, box_color=(0,0,255), border=1, font_color=(0,255,0), draw_strs=None, - fontScale=3, compare_strs=None, draw_location='mid', font_type=None, thickness=2): + cv2.putText( + img=img, + text=text, + org=posi, + fontFace=font, + fontScale=fontScale, + color=color, + thickness=thickness, + lineType=2, + ) + + +def draw_cvrect( + img, + bndboxes, + box_color=(0, 0, 255), + border=1, + font_color=(0, 255, 0), + draw_strs=None, + fontScale=3, + compare_strs=None, + draw_location="mid", + font_type=None, + thickness=2, +): for index, bndbox in enumerate(bndboxes): - bndbox = map(int, bndbox) + bndbox = list(map(int, bndbox)) left_up = (bndbox[0], bndbox[1]) right_bottom = (bndbox[2], bndbox[3]) cv2.rectangle(img, left_up, right_bottom, box_color, border) @@ -232,122 +197,86 @@ def draw_cvrect(img, bndboxes, box_color=(0,0,255), border=1, font_color=(0,255, show_str = draw_strs[index] else: show_str = str(index) - put_cvtext(img, show_str, mid_box, color=font_color, fontScale=fontScale, thickness=thickness, font_type=font_type) + put_cvtext( + img, + show_str, + mid_box, + color=font_color, + fontScale=fontScale, + thickness=thickness, + font_type=font_type, + ) return img -def cv_show(imgs, win_name='show_image'): +def cv_show(imgs, win_name="show_image"): if not isinstance(imgs, list): imgs = [imgs] for index, img in enumerate(imgs): - cv2.imshow(win_name+str(index), img) + cv2.imshow(win_name + str(index), img) cv2.waitKey(0) cv2.destroyAllWindows() def resize_dir_images_with_scale(image_dir, resize_max_edge=224): - image_files = loop(image_dir, ['jpg', 'png']) - for ite_ind, image_file in enumerate(image_files, 1): - img = cv2.imread(image_file) - if img is not None: - if max(img.shape) > 224: - img = resize_with_scale(img, 224) - cv2.imwrite(img, image_file) - else: - shutil.move(image_file) - print("delete broken image file {0}".format(image_file)) - print ("resize dir images max edge to {0}, finished {1}/{2}".format()) + for path in loop(image_dir, IMG_EXTENSIONS): + img = cv2.imread(path) + if img is None: + raise ValueError(f"Cannot read image: {path}") + resized = resize_with_scale(img, resize_max_edge) + if resized is not img and not cv2.imwrite(path, resized): + raise OSError(f"Cannot write image: {path}") def points_to_contour(points): - contours = [[list(p)]for p in points] + contours = [[list(p)] for p in points] return np.asarray(contours, dtype=np.int32) + def points_to_contours(points): return np.asarray([points_to_contour(points)]) + def put_text(img, text, pos, scale=1, color=(255, 255, 255), thickness=1): pos = np.int32(pos) font = cv2.FONT_HERSHEY_SIMPLEX - cv2.putText(img = img, text = text, org = tuple(pos), fontFace = font, - fontScale = scale, color = color, thickness = thickness) - - -def draw_contours(img, contours, idx = -1, color = 1, border_width = 1): - x1,y1,x2,y2,x3,y3,x4,y4 = contours[0].reshape(8, -1) - cv2.line(img, (x1, y1), (x2, y2), (255, 0, 0), thickness=10, lineType=8, shift=0) - for ind, box in enumerate(contours[0]): - cx, cy = box[0] - cv2.circle(img, (cx, cy), 10, (0,0,255), -1) - put_text(img, str(ind+1), (cx ,cy)) + cv2.putText( + img=img, + text=text, + org=tuple(pos), + fontFace=font, + fontScale=scale, + color=color, + thickness=thickness, + ) + + +def draw_contours(img, contours, idx=-1, color=1, border_width=1): cv2.drawContours(img, contours, idx, color, border_width) return img def polly_points2contours(points): - points = [np.reshape(item, [4, 2]) for item in points] - return points_to_contours(points)[0] + return np.asarray(points, dtype=np.int32).reshape(-1, 4, 1, 2) def rename_imfiles_with_md5(imdir, rename_xml=False): - dir_images = loop(imdir) - for ind, im_file in enumerate(dir_images, 1): - with open(im_file, 'r') as f: - contents = f.read() - md5_code = hashlib.md5(contents).hexdigest() - dir_name = os.path.dirname(im_file) - file_type= os.path.basename(im_file)[-4:] - save_path = os.path.join(dir_name, md5_code+file_type) - print("rename file finished {1}/{2}: {0}".format(save_path, ind, len(dir_images))) - os.rename(im_file, save_path) + """Rename images by content hash; preflight and roll back paired XML changes.""" + from pathlib import Path + + from ..file.file_operate import FilesOp + + for image in loop(imdir): + image_path = Path(image) + digest = md5(image_path) + paths = [str(image_path)] if rename_xml: - xml_file = im_file.replace('JPEGImages', 'Annotations').replace(file_type, '.xml') - new_xml = os.path.join(os.path.dirname(xml_file), md5_code+'.xml') - os.rename(xml_file, new_xml) - - -# def compare_with_md5(image_dir, save_dir=None): -# #coding=utf-8 -# import os -# import hashlib -# import shutil - -# input_ = '/data02/heiren/online/2018-12-01-15/' -# output_ = '/data02/heiren/online/2018-12-01-15/md5' -# list_ = {} -# index = 1 - -# for file_ in os.listdir(os.path.join(input_, 'JPEGImages')): -# if index % 500 == 0: -# print index, file_ -# index += 1 - -# fileName = os.path.join(input_, 'JPEGImages', file_) -# data = open(fileName, 'r').read() -# f_md5 = hashlib.md5(data) -# md5_ = f_md5.hexdigest() -# id_ = file_.split('.jpg')[0] - -# xml1 = os.path.join(input_, 'Annotations', id_ + '.xml') -# xml2 = os.path.join(input_, 'Annotations2', id_ + '.xml') -# xml3 = os.path.join(input_, 'Annotations3', id_ + '.xml') -# jpg = os.path.join(input_, 'JPEGImages', id_ + '.jpg') - -# if os.path.exists(xml1): -# if not os.path.exists(os.path.join(output_, 'Annotations')): -# os.makedirs(os.path.join(output_, 'Annotations')) -# shutil.copy(xml1, os.path.join(output_, 'Annotations', md5_ + '.xml')) -# if os.path.exists(xml2): -# if not os.path.exists(os.path.join(output_, 'Annotations2')): -# os.makedirs(os.path.join(output_, 'Annotations2')) -# shutil.copy(xml2, os.path.join(output_, 'Annotations2', md5_ + '.xml')) -# if os.path.exists(xml3): -# if not os.path.exists(os.path.join(output_, 'Annotations3')): -# os.makedirs(os.path.join(output_, 'Annotations3')) -# shutil.copy(xml3, os.path.join(output_, 'Annotations3', md5_ + '.xml')) -# if os.path.exists(jpg): -# if not os.path.exists(os.path.join(output_, 'JPEGImages')): -# os.makedirs(os.path.join(output_, 'JPEGImages')) -# shutil.copy(jpg, os.path.join(output_, 'JPEGImages', md5_ + '.jpg')) - -# pass \ No newline at end of file + parts = list(image_path.parts) + matching = [ + index for index, part in enumerate(parts) if part == "JPEGImages" + ] + if matching: + parts[matching[-1]] = "Annotations" + annotation = Path(*parts).with_suffix(".xml") + paths.append(str(annotation)) + FilesOp.rename_data_pairs(paths, digest) diff --git a/imagedt/tensorflow/__init__.py b/imagedt/tensorflow/__init__.py index bb446f0..7c49784 100644 --- a/imagedt/tensorflow/__init__.py +++ b/imagedt/tensorflow/__init__.py @@ -1,9 +1,19 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -# from . import network -# from . import optim -# from . import trainer -# from . import tools -# from . import lite \ No newline at end of file +"""Compatibility exports, loaded on first use.""" + +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "tools": (".tools", None), + "lite": (".lite", None), + "network": (".network", None), + "optim": (".optim", None), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/tensorflow/lite/__init__.py b/imagedt/tensorflow/lite/__init__.py index 1d8ef10..9412214 100644 --- a/imagedt/tensorflow/lite/__init__.py +++ b/imagedt/tensorflow/lite/__init__.py @@ -1,5 +1 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from .lite_wrapper import * \ No newline at end of file +from .lite_wrapper import Lite_Wrapper as Lite_Wrapper diff --git a/imagedt/tensorflow/lite/lite_wrapper.py b/imagedt/tensorflow/lite/lite_wrapper.py index 760fb8c..69acbdc 100644 --- a/imagedt/tensorflow/lite/lite_wrapper.py +++ b/imagedt/tensorflow/lite/lite_wrapper.py @@ -1,43 +1,41 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Single-input float image classifier using the public Lite interpreter.""" import numpy as np -from tensorflow.contrib.lite.python import interpreter as interpreter_wrapper +from imagedt._backends import require -class Lite_Wrapper(object): - """docstring for Lite_Wrapper""" - def __init__(self, model_path): - super(Lite_Wrapper, self).__init__() - self.model_path = model_path - self.load_model() - self._check_model_type() - def load_model(self): - # load tflite model - self.interpreter = interpreter_wrapper.Interpreter(model_path=self.model_path) - self.interpreter.allocate_tensors() - self.input_details = self.interpreter.get_input_details() - self.output_details = self.interpreter.get_output_details() +class Lite_Wrapper: + def __init__(self, model_path): + self.model_path = model_path + self.load_model() + self._check_model_type() - def _check_model_type(self): - # only support test float tflite model for now - try: - self.input_details[0]['dtype'] == type(np.float32(1.0)) - self.height = input_details[0]['shape'][1] - self.width = input_details[0]['shape'][2] - except Exception as e: - print ("Only support test float tflite model for now, Please check model type!") + def load_model(self): + backend = require("ai_edge_litert.interpreter", "lite") + self.interpreter = backend.Interpreter(model_path=str(self.model_path)) + self.interpreter.allocate_tensors() + self.input_details = self.interpreter.get_input_details() + self.output_details = self.interpreter.get_output_details() - def predict(self, input_data): - """ - input data : should be cv mat array (axis=0) - """ - self.interpreter.set_tensor(self.input_details[0]['index'], input_data) - self.interpreter.invoke() - predictions = np.squeeze(self.interpreter.get_tensor(self.output_details[0]['index'])) - # get top 1 prediction - pre_class = predictions.argsort()[::-1][0] - conf = predictions[pre_class] - return pre_class, conf + def _check_model_type(self): + if len(self.input_details) != 1 or self.input_details[0]["dtype"] != np.float32: + raise ValueError("Only a single float32 input is supported") + shape = self.input_details[0]["shape"] + if len(shape) != 4 or shape[0] != 1: + raise ValueError("Expected a single-image NHWC input") + self.height, self.width = map(int, shape[1:3]) + + def predict(self, input_data): + data = np.asarray(input_data) + if data.dtype != np.float32 or tuple(data.shape) != tuple( + self.input_details[0]["shape"] + ): + raise ValueError("input dtype or shape does not match the model") + self.interpreter.set_tensor(self.input_details[0]["index"], data) + self.interpreter.invoke() + predictions = np.asarray( + self.interpreter.get_tensor(self.output_details[0]["index"]) + ).reshape(-1) + pre_class = int(np.argmax(predictions)) + return pre_class, predictions[pre_class] diff --git a/imagedt/tensorflow/network/__init__.py b/imagedt/tensorflow/network/__init__.py index 6c9486c..441dc17 100644 --- a/imagedt/tensorflow/network/__init__.py +++ b/imagedt/tensorflow/network/__init__.py @@ -1,5 +1 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from .lenet import * \ No newline at end of file +from .lenet import lenet as lenet diff --git a/imagedt/tensorflow/network/lenet.py b/imagedt/tensorflow/network/lenet.py index 88cbcf5..134666b 100644 --- a/imagedt/tensorflow/network/lenet.py +++ b/imagedt/tensorflow/network/lenet.py @@ -1,45 +1,44 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""LeNet with the historical endpoint names and a maintained Keras backend.""" + +from imagedt._backends import require + + +def lenet( + images, + num_classes=10, + is_training=True, + drop_keep_prob=0.5, + prediction_fn=None, + scope="LeNet", +): + tf = require("tensorflow", "tensorflow") + if not 0 < drop_keep_prob <= 1: + raise ValueError("drop_keep_prob must be in (0, 1]") + layers = tf.keras.layers + endpoints = {} + with tf.name_scope(scope): + net = endpoints["conv1"] = layers.Conv2D( + 32, 5, padding="same", activation="relu", name="conv1" + )(images) + net = endpoints["pool1"] = layers.MaxPool2D(2, name="pool1")(net) + net = endpoints["conv2"] = layers.Conv2D( + 64, 5, padding="same", activation="relu", name="conv2" + )(net) + net = endpoints["pool2"] = layers.MaxPool2D(2, name="pool2")(net) + net = endpoints["Flatten"] = layers.Flatten(name="Flatten")(net) + net = endpoints["fc3"] = layers.Dense(1024, activation="relu", name="fc3")(net) + if not num_classes: + return net, endpoints + net = endpoints["dropout3"] = layers.Dropout( + 1 - drop_keep_prob, name="dropout3" + )(net, training=is_training) + logits = endpoints["Logits"] = layers.Dense(num_classes, name="fc4")(net) + endpoints["Predictions"] = ( + tf.nn.softmax(logits, name="Predictions") + if prediction_fn is None + else prediction_fn(logits, scope="Predictions") + ) + return logits, endpoints -import tensorflow as tf - -slim = tf.contrib.slim - -def lenet(images, num_classes=10, is_training=True, drop_keep_prob=0.5, - prediction_fn=slim.softmax, scope='LeNet'): - """ - Args: - images: A batch of `Tensors` of size [batch_size, height, width, channels]. - num_classes: classes of the training dataset - drop_keep_prob: set drop out prob - """ - end_points = {} - - with tf.variable_scope(scope, 'LeNet', [images]): - net = end_points['conv1'] = slim.conv2d(images, 32, [5, 5], scope='conv1') - net = end_points['pool1'] = slim.max_pool2d(net, [2, 2], 2, scope='pool1') - net = end_points['conv2'] = slim.conv2d(net, 64, [5, 5], scope='conv2') - net = end_points['pool2'] = slim.max_pool2d(net, [2, 2], 2, scope='pool2') - net = slim.flatten(net) - end_points['Flatten'] = net - - net = end_points['fc3'] = slim.fully_connected(net, 1024, scope='fc3') - - if not num_classes: - return net, end_points - - net = end_points['dropout3'] = slim.dropout(net, drop_keep_prob, is_training=is_training, - scope='dropout3') - - logits = end_points['Logits'] = slim.fully_connected( - net, num_classes, activation_fn=None, scope='fc4') - - net = end_points['Predictions'] = prediction_fn(logits, scope='Predictions') - - return logits, end_points lenet.default_image_size = 28 - - - diff --git a/imagedt/tensorflow/network/test_lenet.py b/imagedt/tensorflow/network/test_lenet.py index b058fca..4d2eb53 100644 --- a/imagedt/tensorflow/network/test_lenet.py +++ b/imagedt/tensorflow/network/test_lenet.py @@ -12,104 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for slim.nets.vgg.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tensorflow as tf - -from . import lenet - -slim = tf.contrib.slim - - -class LeNetTest(tf.test.TestCase): - print("init test: LeNetTest...") - - def testBuild(self): - batch_size = 5 - height, width = 28, 28 - num_classes = 10 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = lenet(inputs, num_classes) - self.assertEquals(logits.op.name, 'LeNet/fc4/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testFullyConvolutional(self): - batch_size = 1 - height, width = 28, 28 - num_classes = 10 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = lenet(inputs, num_classes) - self.assertEquals(logits.op.name, 'LeNet/fc4/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testGlobalPool(self): - batch_size = 1 - height, width = 28, 28 - num_classes = 10 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = lenet(inputs, num_classes) - self.assertEquals(logits.op.name, 'LeNet/fc4/BiasAdd') - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - - def testEndPoints(self): - batch_size = 5 - height, width = 28, 28 - num_classes = 10 - with self.test_session(): - inputs = tf.random_uniform((batch_size, height, width, 3)) - _, end_points = lenet(inputs, num_classes) - expected_names = ['conv1', - 'pool1', - 'conv2', - 'pool2', - 'Flatten', - 'fc3', - 'dropout3', - 'Logits', - 'Predictions'] - - self.assertSetEqual(set(end_points.keys()), set(expected_names)) - - def testEvaluation(self): - batch_size = 2 - height = width = lenet.default_image_size # set train imgae size - num_classes = 10 - with self.test_session(): - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = lenet(eval_inputs, is_training=False) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - predictions = tf.argmax(logits, 1) - self.assertListEqual(predictions.get_shape().as_list(), [batch_size]) - - def testForward(self): - batch_size = 1 - height = width = lenet.default_image_size # set train imgae size - with self.test_session() as sess: - inputs = tf.random_uniform((batch_size, height, width, 3)) - logits, _ = lenet(inputs) - sess.run(tf.global_variables_initializer()) - output = sess.run(logits) - self.assertTrue(output.any()) - - def TODO_train(self): - imgae_dir = './test/images' - images_path = os.listdir(imgae_dir) - - for image_path in images_path: - cv2.imread(os.path.join(image_dir, image_path)) - - - -if __name__ == '__main__': - tf.test.main() +"""Standalone LeNet compatibility tests; the main suite lives under tests/.""" + +import unittest + +from imagedt._backends import require + +from .lenet import lenet + + +class LeNetTest(unittest.TestCase): + def testBuild(self): + tf = require("tensorflow", "tensorflow") + logits, endpoints = lenet(tf.zeros((2, 28, 28, 3)), 10, is_training=False) + self.assertEqual(tuple(logits.shape), (2, 10)) + self.assertEqual( + set(endpoints), + { + "conv1", + "pool1", + "conv2", + "pool2", + "Flatten", + "fc3", + "dropout3", + "Logits", + "Predictions", + }, + ) + + def testForward(self): + tf = require("tensorflow", "tensorflow") + _, endpoints = lenet(tf.ones((1, 28, 28, 3)), is_training=False) + self.assertAlmostEqual( + float(tf.reduce_sum(endpoints["Predictions"])), 1.0, places=5 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/imagedt/tensorflow/optim/__init__.py b/imagedt/tensorflow/optim/__init__.py index 760c836..e69de29 100644 --- a/imagedt/tensorflow/optim/__init__.py +++ b/imagedt/tensorflow/optim/__init__.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function diff --git a/imagedt/tensorflow/optim/loss.py b/imagedt/tensorflow/optim/loss.py index 5c5c33e..0f3d53d 100644 --- a/imagedt/tensorflow/optim/loss.py +++ b/imagedt/tensorflow/optim/loss.py @@ -1,57 +1,67 @@ -# coding: utf-8 import math -import tensorflow as tf + +from imagedt._backends import tensorflow_v1 + +tf = tensorflow_v1() def smooth_l1_loss(y_true, y_pred): - delta = 0.5 - x = tf.abs(y_true - y_pred) - loss = tf.where(x < delta, 0.5 * x ** 2, delta * (x - 0.5 * delta)) - return loss + delta = 0.5 + x = tf.abs(y_true - y_pred) + loss = tf.where(x < delta, 0.5 * x**2, delta * (x - 0.5 * delta)) + return loss -def arcface_loss(embedding, labels, out_num, w_init=None, s=64., m=0.5): - ''' +def arcface_loss(embedding, labels, out_num, w_init=None, s=64.0, m=0.5): + """ :param embedding: the input embedding vectors :param labels: the input labels, the shape should be eg: (batch_size, 1) :param s: scalar value default is 64 :param out_num: output class num :param m: the margin value, default is 0.5 :return: the final cacualted output, this output is send into the tf.nn.softmax directly - ''' + """ cos_m = math.cos(m) sin_m = math.sin(m) mm = sin_m * m # issue 1 threshold = math.cos(math.pi - m) - with tf.variable_scope('arcface_loss'): + with tf.variable_scope("arcface_loss"): # inputs and weights norm - embedding_norm = tf.norm(embedding, axis=1, keep_dims=True) - embedding = tf.div(embedding, embedding_norm, name='norm_embedding') - weights = tf.get_variable(name='embedding_weights', shape=(embedding.get_shape().as_list()[-1], out_num), - initializer=w_init, dtype=tf.float32) - weights_norm = tf.norm(weights, axis=0, keep_dims=True) - weights = tf.div(weights, weights_norm, name='norm_weights') + embedding = tf.nn.l2_normalize(embedding, axis=1, name="norm_embedding") + weights = tf.get_variable( + name="embedding_weights", + shape=(embedding.get_shape().as_list()[-1], out_num), + initializer=w_init, + dtype=tf.float32, + ) + weights = tf.nn.l2_normalize(weights, axis=0, name="norm_weights") # cos(theta+m) - cos_t = tf.matmul(embedding, weights, name='cos_t') - cos_t2 = tf.square(cos_t, name='cos_2') - sin_t2 = tf.subtract(1., cos_t2, name='sin_2') - sin_t = tf.sqrt(sin_t2, name='sin_t') - cos_mt = s * tf.subtract(tf.multiply(cos_t, cos_m), tf.multiply(sin_t, sin_m), name='cos_mt') + cos_t = tf.matmul(embedding, weights, name="cos_t") + cos_t2 = tf.square(cos_t, name="cos_2") + sin_t2 = tf.maximum(1.0 - cos_t2, 0.0, name="sin_2") + sin_t = tf.sqrt(sin_t2, name="sin_t") + cos_mt = s * tf.subtract( + tf.multiply(cos_t, cos_m), tf.multiply(sin_t, sin_m), name="cos_mt" + ) # this condition controls the theta+m should in range [0, pi] # 0<=theta+m<=pi # -m<=theta<=pi-m cond_v = cos_t - threshold - cond = tf.cast(tf.nn.relu(cond_v, name='if_else'), dtype=tf.bool) + cond = tf.cast(tf.nn.relu(cond_v, name="if_else"), dtype=tf.bool) - keep_val = s*(cos_t - mm) + keep_val = s * (cos_t - mm) cos_mt_temp = tf.where(cond, cos_mt, keep_val) - mask = tf.one_hot(labels, depth=out_num, name='one_hot_mask') + mask = tf.one_hot(tf.reshape(labels, [-1]), depth=out_num, name="one_hot_mask") # mask = tf.squeeze(mask, 1) - inv_mask = tf.subtract(1., mask, name='inverse_mask') + inv_mask = tf.subtract(1.0, mask, name="inverse_mask") - s_cos_t = tf.multiply(s, cos_t, name='scalar_cos_t') + s_cos_t = tf.multiply(s, cos_t, name="scalar_cos_t") - output = tf.add(tf.multiply(s_cos_t, inv_mask), tf.multiply(cos_mt_temp, mask), name='arcface_loss_output') + output = tf.add( + tf.multiply(s_cos_t, inv_mask), + tf.multiply(cos_mt_temp, mask), + name="arcface_loss_output", + ) return output diff --git a/imagedt/tensorflow/tools/DataInterface.py b/imagedt/tensorflow/tools/DataInterface.py index 7d6136d..daae1a2 100644 --- a/imagedt/tensorflow/tools/DataInterface.py +++ b/imagedt/tensorflow/tools/DataInterface.py @@ -1,11 +1,10 @@ -# coding: utf-8 import numpy as np # 数据 -class DataSet(object): +class DataSet: def __init__(self): - super(DataSet, self).__init__() + super().__init__() self._x = None self._y = None self._index_in_epoch = 0 @@ -20,6 +19,8 @@ def y(self): @property def count(self): + if self._x is None: + return 0 if isinstance(self._x, list): return len(self._x) else: @@ -27,6 +28,7 @@ def count(self): def set_x(self, x): self._x = x + self.reset_step() def set_y(self, y): self._y = y @@ -34,41 +36,48 @@ def set_y(self, y): def reset_step(self): self._index_in_epoch = 0 + def _validate_batch(self, batch_size): + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + def batch_count(self, batch_size, isFixed=True): - if (self.count % batch_size) != 0 and not isFixed: - return int(self.count / batch_size) + 1 - else: - return int(self.count / batch_size) + self._validate_batch(batch_size) + return ( + self.count // batch_size + if isFixed + else (self.count + batch_size - 1) // batch_size + ) def next_batch(self, batch_size, isFixed=True): - start = self._index_in_epoch - end = self._index_in_epoch + batch_size - if self._index_in_epoch == 0: - perm = np.arange(self.count) - np.random.shuffle(perm) - if isinstance(self._x, list): - self._x = [self._x[i] for i in perm] - else: - self._x = self._x[perm] - if isinstance(self._y, list): - self._y = [self._y[i] for i in perm] - else: - self._y = self._y[perm] - - self._index_in_epoch += batch_size - - if end > self.count: - if isFixed: - self._index_in_epoch = 0 - return self.next_batch(batch_size, isFixed) - else: - self._index_in_epoch = 0 - end = self.count - - return self._x[start:end], self._y[start:end] + self._validate_batch(batch_size) + if self._y is None or self.count != len(self._y): + raise ValueError("x and y must have equal length") + if self.count == 0 or (isFixed and batch_size > self.count): + raise ValueError("dataset is too small for requested batch") + if self._index_in_epoch >= self.count or ( + isFixed and self._index_in_epoch + batch_size > self.count + ): + self.reset_step() + if self._index_in_epoch == 0: + perm = np.random.permutation(self.count) + self._x = ( + [self._x[i] for i in perm] + if isinstance(self._x, list) + else self._x[perm] + ) + self._y = ( + [self._y[i] for i in perm] + if isinstance(self._y, list) + else self._y[perm] + ) + start = self._index_in_epoch + end = min(start + batch_size, self.count) + self._index_in_epoch = end + return self._x[start:end], self._y[start:end] + # 数据集 -class DataSets(object): +class DataSets: def __init__(self): self._train = DataSet() self._validate = DataSet() @@ -91,7 +100,7 @@ def train(self): @property def x_shape(self): if self._x_shape is None: - raise ValueError('invalid argument') + raise ValueError("invalid argument") return self._x_shape def set_x_shape(self, x_shape): @@ -100,7 +109,7 @@ def set_x_shape(self, x_shape): @property def y_shape(self): if self._y_shape is None: - raise ValueError('invalid argument') + raise ValueError("invalid argument") return self._y_shape def set_y_shape(self, y_shape): diff --git a/imagedt/tensorflow/tools/Records.py b/imagedt/tensorflow/tools/Records.py index 554d420..b2668e7 100644 --- a/imagedt/tensorflow/tools/Records.py +++ b/imagedt/tensorflow/tools/Records.py @@ -1,187 +1,262 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - import os +import random +import re import sys +import tempfile +import traceback +from pathlib import Path + import cv2 -import random import numpy as np -import traceback -import tensorflow as tf -from . import dataset_util +import imagedt +from imagedt._backends import tensorflow_v1 + from ...file.file_operate import FilesOp from ...file.parse_annotation import Anno_OP -from .data_provider import tf_noise_padd -import imagedt +from . import dataset_util + +tf = tensorflow_v1() -os.environ['CUDA_VISIBLE_DEVICES'] = '0' -class ImageReader(object): - """Helper class that provides TensorFlow image coding utilities.""" - def __init__(self): - # Initializes function that decodes RGB JPEG data. - self._decode_jpeg_data = tf.placeholder(dtype=tf.string) - self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3) +class ImageReader: + """Helper class that provides TensorFlow image coding utilities.""" - def read_image_dims(self, sess, image_data): - image = self.decode_jpeg(sess, image_data) - return image.shape[0], image.shape[1] + def __init__(self): + pass - def decode_jpeg(self, sess, image_data): - image = sess.run(self._decode_jpeg, - feed_dict={self._decode_jpeg_data: image_data}) - assert len(image.shape) == 3 - assert image.shape[2] == 3 - return image + def read_image_dims(self, sess, image_data): + return self.decode_jpeg(sess, image_data).shape[:2] + + def decode_jpeg(self, sess, image_data): + image = cv2.imdecode( + np.frombuffer(image_data, dtype=np.uint8), cv2.IMREAD_COLOR + ) + if image is None: + raise ValueError("Cannot decode image") + return cv2.cvtColor(image, cv2.COLOR_BGR2RGB) class Record_Writer(ImageReader): - """docstring for RecordWriter""" - def __init__(self): - super(Record_Writer, self).__init__() - self.error_images = [] - - def image_to_tfexample(self, image_data, image_format, height, width, class_id): - return tf.train.Example(features=tf.train.Features( - feature={'image/encoded': dataset_util.bytes_feature(image_data), - 'image/format': dataset_util.bytes_feature(image_format), - 'image/class/label': dataset_util.int64_feature(class_id), - 'image/height': dataset_util.int64_feature(height), - 'image/width': dataset_util.int64_feature(width), })) - - def convert_to_tfrecord(self, f_lines, svae_dir, dataset_type='train'): - """Converts a file to TFRecords.""" - print('Generating TFRecords......' ) - with tf.Session() as sess: - piece_count = 50000 - num_pieces = int(len(f_lines) / piece_count + 1) - for num_piece in range(num_pieces): - start_id, end_id = num_piece*piece_count, min(len(f_lines), (num_piece+1)*piece_count) - - output_file = os.path.join(svae_dir, dataset_type + str(num_piece+1).zfill(6) + '.tfrecord') - with tf.python_io.TFRecordWriter(output_file) as record_writer: - for index in range(start_id, end_id): - try: - img = cv2.imread(f_lines[index][0]) - height, width, chanel = img.shape - encoded_image = cv2.imencode('.png', img)[1].tostring() - # encoded_image = tf.gfile.FastGFile(f_lines[index][0], 'rb').read() - except Exception as e: - traceback.print_exc(file=sys.stdout) - print("error image file {0}...".format(os.path.basename(f_lines[index][0]))) - self.error_images.append(os.path.basename(f_lines[index][0])) - continue - - # tf example format: NCHW - image_format = os.path.basename(f_lines[index][0]).split('.')[-1].encode() - example = self.image_to_tfexample(encoded_image, image_format, height, width, int(f_lines[index][1])) - record_writer.write(example.SerializeToString()) - print("finished: ", index + 1, '/', len(f_lines), "; image height: {0}, width: {1}".format(height, width)) - - sys.stdout.write('\n') - sys.stdout.flush() - - def map_int(self, m_list): - return np.array(map(int, map(float, m_list))) - - def create_tf_example(self, sess, jpg_path, anno_infos): - with tf.gfile.GFile(jpg_path, 'rb') as fid: - encoded_image = fid.read() - # get infos - chanel = 3 # now set 3 - height, width = self.read_image_dims(sess, encoded_image) - image_type = os.path.basename(jpg_path).split('.')[-1] - image_format = image_type.encode() - - # convert infos - anno_infos = np.array(anno_infos) - xmins = self.map_int(anno_infos[:, 0]) / float(width) # List of normalized left x coordinates in bounding box (1 per box) - ymins = self.map_int(anno_infos[:, 1]) / float(height) # List of normalized top y coordinates in bounding box (1 per box) - xmaxs = self.map_int(anno_infos[:, 2]) / float(width) # List of normalized right x coordinates in bounding box # (1 per box) - ymaxs = self.map_int(anno_infos[:, 3]) / float(height) # List of normalized bottom y coordinates in bounding box # (1 per box) - classes = self.map_int(anno_infos[:, 4]) # List of integer class id of bounding box (1 per box) - classes_text = anno_infos[:, 5] # List of string class name of bounding box (1 per box) - - assert len(xmins) == len(ymins) == len(xmaxs) == len(ymaxs) == len(classes) == len(classes_text) - - filename = jpg_path.encode() # Filename of the image. Empty if image is not from file - tf_example = tf.train.Example(features=tf.train.Features( - feature={'image/height': dataset_util.int64_feature(height), - 'image/width': dataset_util.int64_feature(width), - 'image/channels': dataset_util.int64_feature(chanel), - 'image/filename': dataset_util.bytes_feature(filename), - 'image/source_id': dataset_util.bytes_feature(filename), - 'image/encoded': dataset_util.bytes_feature(encoded_image), - 'image/format': dataset_util.bytes_feature(image_format), - 'image/object/bbox/xmin': dataset_util.float_list_feature(xmins), - 'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs), - 'image/object/bbox/ymin': dataset_util.float_list_feature(ymins), - 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs), - 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), - 'image/object/class/label': dataset_util.int64_list_feature(classes), })) - return tf_example - - def write_cls_records(self, train_images_file, save_dir, test_ratio=0): - with open(train_images_file, 'r') as f: - all_files = [item.strip().split('\t') for item in f.readlines()] - # save images file in tfrecord dir - imagedt.file.write_csv(all_files, os.path.join(save_dir,'image_files.txt')) - - # shuffle all the files - random.shuffle(all_files) - sample_count = len(all_files) - # read file, write to tfrecords - test_count = int(sample_count * test_ratio) - if test_count: - test_samples = all_files[-test_count:] - self.convert_to_tfrecord(test_samples, save_dir, dataset_type='validation_') - # save error images - imagedt.file.write_txt(self.error_images, os.path.join(save_dir,'test_error_images.txt')) - - train_samples = all_files[:sample_count-test_count] - self.convert_to_tfrecord(all_files, save_dir, dataset_type='train_') - # save error images - imagedt.file.write_txt(self.error_images, os.path.join(save_dir,'train_error_images.txt')) - - def converte_anno_info(self, data_dir, det_cls_name='3488'): - data_pairs = FilesOp.get_jpg_xml_pairs(data_dir) - examples = {} - for index, data_pair in enumerate(data_pairs): - jpg_path, xml_path = data_pair - anno_infos = Anno_OP.parse_lab_xml(xml_path) - # add default class and class_name - anno_infos = [item+['3488']*2 for item in anno_infos] - examples[jpg_path] = anno_infos - print("Read image pairs: ", len(examples)) - return examples - - def write_detect_records(self, data_dir, save_dir, record_name='traning_detect.tfrecord'): - examples = self.converte_anno_info(data_dir) - record_name = os.path.join(save_dir, record_name) - writer = tf.python_io.TFRecordWriter(record_name) - with tf.Session() as sess: - for index, key_path in enumerate(examples): - tf_example = self.create_tf_example(sess, key_path, examples[key_path]) - writer.write(tf_example.SerializeToString()) - print("write record files: {0}/{1} ".format(index+1, len(examples))) - writer.close() - - print("All class: 3488, class_name 3488, records save dir {}".format(save_dir)) + """docstring for RecordWriter""" + + def __init__(self): + super().__init__() + self.error_images = [] + + def image_to_tfexample(self, image_data, image_format, height, width, class_id): + return tf.train.Example( + features=tf.train.Features( + feature={ + "image/encoded": dataset_util.bytes_feature(image_data), + "image/format": dataset_util.bytes_feature(image_format), + "image/class/label": dataset_util.int64_feature(class_id), + "image/height": dataset_util.int64_feature(height), + "image/width": dataset_util.int64_feature(width), + } + ) + ) + + def convert_to_tfrecord(self, f_lines, svae_dir, dataset_type="train"): + """Converts a file to TFRecords.""" + os.makedirs(svae_dir, exist_ok=True) + print("Generating TFRecords......") + with tf.Session(): + piece_count = 50000 + num_pieces = (len(f_lines) + piece_count - 1) // piece_count + for num_piece in range(num_pieces): + start_id, end_id = ( + num_piece * piece_count, + min(len(f_lines), (num_piece + 1) * piece_count), + ) + + output_file = os.path.join( + svae_dir, dataset_type + str(num_piece + 1).zfill(6) + ".tfrecord" + ) + with tf.python_io.TFRecordWriter(output_file) as record_writer: + for index in range(start_id, end_id): + try: + img = cv2.imread(f_lines[index][0]) + height, width, chanel = img.shape + encoded_image = cv2.imencode(".png", img)[1].tobytes() + # encoded_image = tf.gfile.FastGFile(f_lines[index][0], 'rb').read() + except Exception: + traceback.print_exc(file=sys.stdout) + print( + f"error image file {os.path.basename(f_lines[index][0])}..." + ) + self.error_images.append( + os.path.basename(f_lines[index][0]) + ) + continue + + # tf example format: NCHW + image_format = b"png" + example = self.image_to_tfexample( + encoded_image, + image_format, + height, + width, + int(f_lines[index][1]), + ) + record_writer.write(example.SerializeToString()) + print( + "finished: ", + index + 1, + "/", + len(f_lines), + f"; image height: {height}, width: {width}", + ) + + sys.stdout.write("\n") + sys.stdout.flush() + + def map_int(self, m_list): + return np.array(list(map(int, list(map(float, m_list))))) + + def create_tf_example(self, sess, jpg_path, anno_infos): + with tf.gfile.GFile(jpg_path, "rb") as fid: + encoded_image = fid.read() + # get infos + chanel = 3 # now set 3 + height, width = self.read_image_dims(sess, encoded_image) + image_type = os.path.basename(jpg_path).split(".")[-1] + image_format = image_type.encode() + + # convert infos + anno_infos = np.asarray(anno_infos).reshape(-1, 6) + xmins = self.map_int(anno_infos[:, 0]) / float( + width + ) # List of normalized left x coordinates in bounding box (1 per box) + ymins = self.map_int(anno_infos[:, 1]) / float( + height + ) # List of normalized top y coordinates in bounding box (1 per box) + xmaxs = self.map_int(anno_infos[:, 2]) / float( + width + ) # List of normalized right x coordinates in bounding box # (1 per box) + ymaxs = self.map_int(anno_infos[:, 3]) / float( + height + ) # List of normalized bottom y coordinates in bounding box # (1 per box) + classes = self.map_int( + anno_infos[:, 4] + ) # List of integer class id of bounding box (1 per box) + classes_text = anno_infos[ + :, 5 + ] # List of string class name of bounding box (1 per box) + + assert ( + len(xmins) + == len(ymins) + == len(xmaxs) + == len(ymaxs) + == len(classes) + == len(classes_text) + ) + + filename = ( + jpg_path.encode() + ) # Filename of the image. Empty if image is not from file + tf_example = tf.train.Example( + features=tf.train.Features( + feature={ + "image/height": dataset_util.int64_feature(height), + "image/width": dataset_util.int64_feature(width), + "image/channels": dataset_util.int64_feature(chanel), + "image/filename": dataset_util.bytes_feature(filename), + "image/source_id": dataset_util.bytes_feature(filename), + "image/encoded": dataset_util.bytes_feature(encoded_image), + "image/format": dataset_util.bytes_feature(image_format), + "image/object/bbox/xmin": dataset_util.float_list_feature(xmins), + "image/object/bbox/xmax": dataset_util.float_list_feature(xmaxs), + "image/object/bbox/ymin": dataset_util.float_list_feature(ymins), + "image/object/bbox/ymax": dataset_util.float_list_feature(ymaxs), + "image/object/class/text": dataset_util.bytes_list_feature( + classes_text + ), + "image/object/class/label": dataset_util.int64_list_feature( + classes + ), + } + ) + ) + return tf_example + + def write_cls_records(self, train_images_file, save_dir, test_ratio=0): + if not 0 <= test_ratio <= 1: + raise ValueError("test_ratio must be in [0, 1]") + os.makedirs(save_dir, exist_ok=True) + self.error_images = [] + with open(train_images_file, encoding="utf-8") as f: + all_files = [item.strip().split("\t") for item in f.readlines()] + # save images file in tfrecord dir + imagedt.file.write_csv(all_files, os.path.join(save_dir, "image_files.txt")) + + # shuffle all the files + random.shuffle(all_files) + sample_count = len(all_files) + # Generate before replacing the owned output family, so failed writes + # leave the previous dataset usable. Publishing multiple files is not atomic. + test_count = int(sample_count * test_ratio) + with tempfile.TemporaryDirectory(dir=save_dir) as staging: + self.convert_to_tfrecord( + all_files[sample_count - test_count :], + staging, + dataset_type="validation_", + ) + imagedt.file.write_txt( + self.error_images, os.path.join(staging, "test_error_images.txt") + ) + self.error_images = [] + self.convert_to_tfrecord( + all_files[: sample_count - test_count], staging, dataset_type="train_" + ) + imagedt.file.write_txt( + self.error_images, os.path.join(staging, "train_error_images.txt") + ) + generated = {path.name for path in Path(staging).iterdir()} + for path in Path(staging).iterdir(): + os.replace(path, Path(save_dir) / path.name) + for path in Path(save_dir).iterdir(): + if ( + re.fullmatch(r"(?:train|validation)_[0-9]{6}\.tfrecord", path.name) + and path.name not in generated + ): + path.unlink() + + def converte_anno_info(self, data_dir, det_cls_name="3488"): + data_pairs = FilesOp.get_jpg_xml_pairs(data_dir) + examples = {} + for data_pair in data_pairs: + jpg_path, xml_path = data_pair + anno_infos = Anno_OP.parse_lab_xml(xml_path) + # add default class and class_name + anno_infos = [item + [str(det_cls_name)] * 2 for item in anno_infos] + examples[jpg_path] = anno_infos + print("Read image pairs: ", len(examples)) + return examples + + def write_detect_records( + self, data_dir, save_dir, record_name="traning_detect.tfrecord" + ): + examples = self.converte_anno_info(data_dir) + record_name = os.path.join(save_dir, record_name) + os.makedirs(save_dir, exist_ok=True) + with tf.python_io.TFRecordWriter(record_name) as writer: + for key_path in examples: + example = self.create_tf_example(None, key_path, examples[key_path]) + writer.write(example.SerializeToString()) + + print(f"All class: 3488, class_name 3488, records save dir {save_dir}") RecordWriter = Record_Writer() -from tensorflow.python.platform import gfile def write_pbmodel_summery(tf_pbmodel, log_dir): - with tf.Session() as sess: - model_filename =tf_pbmodel - with gfile.FastGFile(model_filename, 'rb') as f: - graph_def = tf.GraphDef() - graph_def.ParseFromString(f.read()) - g_in = tf.import_graph_def(graph_def) - train_writer = tf.summary.FileWriter(log_dir) - train_writer.add_graph(sess.graph) \ No newline at end of file + graph = tf.Graph() + with graph.as_default(), tf.gfile.GFile(tf_pbmodel, "rb") as stream: + definition = tf.GraphDef() + definition.ParseFromString(stream.read()) + tf.import_graph_def(definition, name="") + with graph.as_default(), tf.summary.FileWriter(log_dir, graph=graph) as writer: + writer.flush() diff --git a/imagedt/tensorflow/tools/__init__.py b/imagedt/tensorflow/tools/__init__.py index fb10c96..9283e4f 100644 --- a/imagedt/tensorflow/tools/__init__.py +++ b/imagedt/tensorflow/tools/__init__.py @@ -1,9 +1,37 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from .tfmodel_wrapper import * -from .dataset_util import * -from .DataInterface import * -from .data_provider import * -from .Records import * \ No newline at end of file +"""Compatibility exports, loaded on first use.""" + +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "TFmodel_Wrapper": (".tfmodel_wrapper", "TFmodel_Wrapper"), + "int64_list_feature": (".dataset_util", "int64_list_feature"), + "bytes_list_feature": (".dataset_util", "bytes_list_feature"), + "float_list_feature": (".dataset_util", "float_list_feature"), + "int64_feature": (".dataset_util", "int64_feature"), + "bytes_feature": (".dataset_util", "bytes_feature"), + "float_feature": (".dataset_util", "float_feature"), + "read_examples_list": (".dataset_util", "read_examples_list"), + "recursive_parse_xml_to_dict": (".dataset_util", "recursive_parse_xml_to_dict"), + "make_initializable_iterator": (".dataset_util", "make_initializable_iterator"), + "read_dataset": (".dataset_util", "read_dataset"), + "DataSet": (".DataInterface", "DataSet"), + "DataSets": (".DataInterface", "DataSets"), + "dynamically_loaded_data": (".data_provider", "dynamically_loaded_data"), + "resize_image_keep_aspect": (".data_provider", "resize_image_keep_aspect"), + "tf_noise_padd": (".data_provider", "tf_noise_padd"), + "tf_JitterCut": (".data_provider", "tf_JitterCut"), + "Data_Provider": (".data_provider", "Data_Provider"), + "ImageReader": (".Records", "ImageReader"), + "Record_Writer": (".Records", "Record_Writer"), + "RecordWriter": (".Records", "RecordWriter"), + "write_pbmodel_summery": (".Records", "write_pbmodel_summery"), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/tensorflow/tools/data_provider.py b/imagedt/tensorflow/tools/data_provider.py index cd62880..ecdba54 100644 --- a/imagedt/tensorflow/tools/data_provider.py +++ b/imagedt/tensorflow/tools/data_provider.py @@ -1,181 +1,142 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import cv2 import numpy as np -import tensorflow as tf import imagedt -from .DataInterface import * -from ...image.process import noise_padd +from imagedt._backends import tensorflow_v1 + +from ...image.process import padd_pixel +from .DataInterface import DataSet as DataSet +from .DataInterface import DataSets as DataSets -slim = tf.contrib.slim +tf = tensorflow_v1() def dynamically_loaded_data(image_paths, labels, height=224, width=224): - labels = map(int, labels) - # max(x. 128): prevent memory bomb - images = np.zeros([max(len(image_paths), 128), height, width, 3], np.float32) - for index, image_path in enumerate(image_paths): - cvmat = cv2.imread(image_path) - h, w, c = cvmat.shape - if h != 224 or w != 224: - cvmat = noise_padd(cvmat, edge_size=224,start_pixel_value=0) - images[index] = np.array(cvmat) - return images, labels + if len(image_paths) != len(labels): + raise ValueError("image paths and labels must have equal length") + images = np.empty((len(image_paths), height, width, 3), dtype=np.float32) + for index, path in enumerate(image_paths): + image = cv2.imread(path) + if image is None: + raise ValueError(f"Cannot read image: {path}") + images[index] = padd_pixel(image, width, height) + return images, list(map(int, labels)) def resize_image_keep_aspect(image, max_edge=224): - # Take width/height - initial_height = tf.shape(image)[0] - initial_width = tf.shape(image)[1] - - # Take the greater value, and use it for the ratio - max_value = tf.maximum(initial_width, initial_height) - ratio = tf.to_float(max_value) / tf.constant(max_edge, dtype=tf.float32) - - new_width = tf.to_int32(tf.to_float(initial_width) / ratio) - new_height = tf.to_int32(tf.to_float(initial_height) / ratio) - - def set_w(): - new_width = max_edge - return new_width, new_height - - def set_h(): - new_height = max_edge - return new_width, new_height - - new_width, new_height = tf.cond(tf.greater(new_width, new_height), set_w, set_h) - - return tf.image.resize_images(image, [max_edge, new_width]) + if max_edge < 1: + raise ValueError("max_edge must be positive") + shape = tf.shape(image)[:2] + ratio = tf.cast(max_edge, tf.float32) / tf.cast(tf.reduce_max(shape), tf.float32) + target = tf.maximum( + 1, tf.cast(tf.round(tf.cast(shape, tf.float32) * ratio), tf.int32) + ) + return tf.image.resize_images(image, target) def tf_noise_padd(images, max_edge=224, start_pixel=0, end_pixel=255): - # resize image with scale - images = resize_image_keep_aspect(images, max_edge) - - height = tf.shape(images)[0] - width = tf.shape(images)[1] - channels = 3 - - # # height > width - def case_height_width(): - left_pad_size = tf.div(tf.subtract(max_edge, width), 2) - right_pad_size = tf.subtract(tf.subtract(max_edge, width), left_pad_size) - noise_left = tf.random_uniform((height, left_pad_size, channels), minval=start_pixel, - maxval=end_pixel,dtype=tf.float32) - noise_right = tf.random_uniform((height, right_pad_size, channels), minval=start_pixel, - maxval=end_pixel, dtype=tf.float32) - # noise_left = tf.ones((height, left_pad_size, channels)) * start_pixel - # noise_right = tf.ones((height, right_pad_size, channels)) * start_pixel - merge = tf.concat([noise_left, images, noise_right], axis=1) - return merge - - # width > height - def case_width_height(): - top_padd_size = tf.div(tf.subtract(max_edge, height), 2) - bottom_padd_size = tf.subtract(tf.subtract(max_edge, height), top_padd_size) - noise_top = tf.random_uniform((top_padd_size, width, channels), minval=start_pixel, - maxval=end_pixel, dtype=tf.float32) - noise_bottom = tf.random_uniform((bottom_padd_size, width, channels), minval=start_pixel, - maxval=end_pixel, dtype=tf.float32) - # noise_top = tf.ones((top_padd_size, width, channels)) * start_pixel - # noise_bottom = tf.ones((bottom_padd_size, width, channels)) * start_pixel - merge = tf.concat([noise_top, images, noise_bottom], axis=0) - return merge - - padd_noise_op = tf.cond(tf.greater(height, width), case_height_width, case_width_height) - return padd_noise_op + if not 0 <= start_pixel < end_pixel <= 256: + raise ValueError("invalid pixel range") + images = resize_image_keep_aspect(images, max_edge) + shape = tf.shape(images) + top = (max_edge - shape[0]) // 2 + left = (max_edge - shape[1]) // 2 + paddings = [ + [top, max_edge - shape[0] - top], + [left, max_edge - shape[1] - left], + [0, 0], + ] + padded = tf.pad(images, paddings) + mask = tf.pad(tf.ones_like(images), paddings) + noise = tf.random_uniform([max_edge, max_edge, shape[2]], start_pixel, end_pixel) + return padded + noise * (1 - mask) def tf_JitterCut(images, jitter=0.05): - width, height ,channels = images.shape - ratio = tf.random_uniform(1, minval=0, maxval=jitter, dtype=tf.float32) - new_h = height*(1-ratio) - new_w = width*(1-ratio) - - start_x = tf.random_uniform(0, minval=0, maxval=width-new_w) - start_y = tf.random_uniform(0, minval=0, maxval=height-new_h) - - return images[start_y:new_h, start_x:new_w] - - # cv::Mat JitterCut(cv::Mat &src, float &jitter){ - # cv::Mat cv_img = src.clone(); - # uint64 timeseed =(double)cv::getTickCount(); - # cv::RNG rng(timeseed); - # unsigned int height = cv_img.rows; - # unsigned int width = cv_img.cols; - # float ratio = rng.uniform(0., jitter); - # unsigned int new_h = height*(1-ratio); - # unsigned int new_w = width*(1-ratio); - # unsigned int start_x = rng.uniform(0, width-new_w); - # unsigned int start_y = rng.uniform(0, height-new_h); - # cv::Rect roi(start_x,start_y,new_w,new_h); - # cv_img = cv_img(roi); - # return cv_img; - # } - - -class Data_Provider(object): - """docstring for Data_Provider""" - def __init__(self, data_dir, num_classes): - super(Data_Provider, self).__init__() - self.data_dir = data_dir - self.num_classes = num_classes - self._init_dataset_infos() - self._init_reader() - - @property - def _get_tfrecords(self, file_format=['.tfrecord']): - return imagedt.dir.loop(self.data_dir, file_format) - - def _search_laebls_file(self): - sear_files = imagedt.dir.loop(self.data_dir, ['.txt']) - sear_label_file = [item for item in sear_files if ('label' in item) - and item.endswith('.txt')] - if sear_label_file: - lines = imagedt.file.readlines(sear_label_file[0]) - self.num_classes = len(lines) - - def _init_dataset_infos(self): - self._search_laebls_file() - - @property - def _log_dataset_infos(self): - print('##'*20) - print ("Dataset infos:\ndata_dir: {0} \nnum_classes: {1}".format(self.data_dir, self.num_classes)) - print('##'*20) - - def _init_reader(self): - # read all tfrecord files - self.reader = tf.TFRecordReader() - record_files = self._get_tfrecords - self.filequeue = tf.train.string_input_producer(record_files) - - - def read_from_tfrecord(self, batch_size=32, image_shape=(224, 224, 3)): - self.image_shape = image_shape - _, fetch_tensors = self.reader.read(self.filequeue) - load_features = tf.parse_single_example( - fetch_tensors,features={ - 'image/height': tf.FixedLenFeature([], dtype=tf.int64), - 'image/width': tf.FixedLenFeature([], dtype=tf.int64), - 'image/class/label': tf.FixedLenFeature([], dtype=tf.int64), - 'image/encoded': tf.FixedLenFeature([], dtype=tf.string), - } + if not 0 <= jitter < 1: + raise ValueError("jitter must be in [0, 1)") + shape = tf.shape(images) + ratio = tf.random_uniform([], 0, jitter) if jitter else tf.constant(0.0) + target = tf.maximum( + 1, tf.cast(tf.cast(shape[:2], tf.float32) * (1 - ratio), tf.int32) ) - height = tf.cast(load_features['image/height'], tf.int32) - width = tf.cast(load_features['image/width'], tf.int32) - label = tf.cast(load_features['image/class/label'], tf.int64) - image = tf.image.decode_jpeg(load_features['image/encoded'], channels=image_shape[2]) - image = tf_noise_padd(image, max_edge=image_shape[0], start_pixel=255) - image = tf.reshape(image, image_shape, name=None) - # make a batch - label = slim.one_hot_encoding(label, self.num_classes) - image_batch, label_batch = tf.train.shuffle_batch([image, label], - batch_size=batch_size, - capacity=500, - min_after_dequeue=100, - num_threads=4) - return image_batch, label_batch \ No newline at end of file + return tf.random_crop(images, [target[0], target[1], shape[2]]) + + +class Data_Provider: + """docstring for Data_Provider""" + + def __init__(self, data_dir, num_classes): + super().__init__() + self.data_dir = data_dir + self.num_classes = num_classes + self._init_dataset_infos() + self._init_reader() + + @property + def _get_tfrecords(self, file_format=(".tfrecord",)): + return imagedt.dir.loop(self.data_dir, file_format) + + def _search_laebls_file(self): + sear_files = imagedt.dir.loop(self.data_dir, [".txt"]) + sear_label_file = [ + item for item in sear_files if ("label" in item) and item.endswith(".txt") + ] + if sear_label_file: + lines = imagedt.file.readlines(sear_label_file[0]) + self.num_classes = len(lines) + + def _init_dataset_infos(self): + self._search_laebls_file() + + @property + def _log_dataset_infos(self): + print("##" * 20) + print( + f"Dataset infos:\ndata_dir: {self.data_dir} \nnum_classes: {self.num_classes}" + ) + print("##" * 20) + + def _init_reader(self): + # read all tfrecord files + self.reader = tf.TFRecordReader() + record_files = self._get_tfrecords + self.filequeue = tf.train.string_input_producer(record_files) + + def read_from_tfrecord(self, batch_size=32, image_shape=(224, 224, 3)): + self.image_shape = image_shape + _, fetch_tensors = self.reader.read(self.filequeue) + load_features = tf.parse_single_example( + fetch_tensors, + features={ + "image/height": tf.FixedLenFeature([], dtype=tf.int64), + "image/width": tf.FixedLenFeature([], dtype=tf.int64), + "image/class/label": tf.FixedLenFeature([], dtype=tf.int64), + "image/encoded": tf.FixedLenFeature([], dtype=tf.string), + }, + ) + label = tf.cast(load_features["image/class/label"], tf.int64) + image = tf.image.decode_image( + load_features["image/encoded"], + channels=image_shape[2], + expand_animations=False, + ) + image.set_shape([None, None, image_shape[2]]) + image = tf.image.resize_image_with_pad(image, image_shape[0], image_shape[1]) + image.set_shape(image_shape) + # make a batch + with tf.control_dependencies( + [ + tf.assert_greater_equal(label, tf.constant(0, dtype=tf.int64)), + tf.assert_less(label, tf.cast(self.num_classes, tf.int64)), + ] + ): + label = tf.one_hot(label, self.num_classes) + image_batch, label_batch = tf.train.shuffle_batch( + [image, label], + batch_size=batch_size, + capacity=500, + min_after_dequeue=100, + num_threads=4, + ) + return image_batch, label_batch diff --git a/imagedt/tensorflow/tools/dataset_util.py b/imagedt/tensorflow/tools/dataset_util.py index 69038a4..20d7c61 100644 --- a/imagedt/tensorflow/tools/dataset_util.py +++ b/imagedt/tensorflow/tools/dataset_util.py @@ -1,168 +1,181 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - """Utility functions for creating TFRecord data sets.""" -import tensorflow as tf +from imagedt._backends import tensorflow_v1 + +tf = tensorflow_v1() -def int64_list_feature(value): - return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) +def int64_list_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) -def bytes_list_feature(value): - return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) +def bytes_list_feature(value): + return tf.train.Feature( + bytes_list=tf.train.BytesList( + value=[v.encode("utf-8") if isinstance(v, str) else v for v in value] + ) + ) -def float_list_feature(value): - return tf.train.Feature(float_list=tf.train.FloatList(value=value)) +def float_list_feature(value): + return tf.train.Feature(float_list=tf.train.FloatList(value=value)) def int64_feature(values): - """Returns a TF-Feature of int64s. + """Returns a TF-Feature of int64s. - Args: - values: A scalar or list of values. + Args: + values: A scalar or list of values. - Returns: - A TF-Feature. - """ - if not isinstance(values, (tuple, list)): - values = [values] - return tf.train.Feature(int64_list=tf.train.Int64List(value=values)) + Returns: + A TF-Feature. + """ + if not isinstance(values, (tuple, list)): + values = [values] + return tf.train.Feature(int64_list=tf.train.Int64List(value=values)) def bytes_feature(values): - """Returns a TF-Feature of bytes. + """Returns a TF-Feature of bytes. - Args: - values: A string. + Args: + values: A string. - Returns: - A TF-Feature. - """ - return tf.train.Feature(bytes_list=tf.train.BytesList(value=[values])) + Returns: + A TF-Feature. + """ + return tf.train.Feature( + bytes_list=tf.train.BytesList( + value=[values.encode("utf-8") if isinstance(values, str) else values] + ) + ) def float_feature(values): - """Returns a TF-Feature of floats. - - Args: - values: A scalar of list of values. + """Returns a TF-Feature of floats. - Returns: - A TF-Feature. - """ - if not isinstance(values, (tuple, list)): - values = [values] - return tf.train.Feature(float_list=tf.train.FloatList(value=values)) + Args: + values: A scalar of list of values. + Returns: + A TF-Feature. + """ + if not isinstance(values, (tuple, list)): + values = [values] + return tf.train.Feature(float_list=tf.train.FloatList(value=values)) def read_examples_list(path): - """Read list of training or validation examples. + """Read list of training or validation examples. - The file is assumed to contain a single example per line where the first - token in the line is an identifier that allows us to find the image and - annotation xml for that example. + The file is assumed to contain a single example per line where the first + token in the line is an identifier that allows us to find the image and + annotation xml for that example. - For example, the line: - xyz 3 - would allow us to find files xyz.jpg and xyz.xml (the 3 would be ignored). + For example, the line: + xyz 3 + would allow us to find files xyz.jpg and xyz.xml (the 3 would be ignored). - Args: - path: absolute path to examples list file. + Args: + path: absolute path to examples list file. - Returns: - list of example identifiers (strings). - """ - with tf.gfile.GFile(path) as fid: - lines = fid.readlines() - return [line.strip().split(' ')[0] for line in lines] + Returns: + list of example identifiers (strings). + """ + with tf.gfile.GFile(path) as fid: + lines = fid.readlines() + return [line.strip().split(" ")[0] for line in lines] def recursive_parse_xml_to_dict(xml): - """Recursively parses XML contents to python dict. - - We assume that `object` tags are the only ones that can appear - multiple times at the same level of a tree. - - Args: - xml: xml tree obtained by parsing XML file contents using lxml.etree - - Returns: - Python dictionary holding XML contents. - """ - if not xml: - return {xml.tag: xml.text} - result = {} - for child in xml: - child_result = recursive_parse_xml_to_dict(child) - if child.tag != 'object': - result[child.tag] = child_result[child.tag] - else: - if child.tag not in result: - result[child.tag] = [] - result[child.tag].append(child_result[child.tag]) - return {xml.tag: result} + """Recursively parses XML contents to python dict. + + We assume that `object` tags are the only ones that can appear + multiple times at the same level of a tree. + + Args: + xml: xml tree obtained by parsing XML file contents using lxml.etree + + Returns: + Python dictionary holding XML contents. + """ + if len(xml) == 0: + return {xml.tag: xml.text} + result = {} + for child in xml: + child_result = recursive_parse_xml_to_dict(child) + if child.tag != "object": + result[child.tag] = child_result[child.tag] + else: + if child.tag not in result: + result[child.tag] = [] + result[child.tag].append(child_result[child.tag]) + return {xml.tag: result} def make_initializable_iterator(dataset): - """Creates an iterator, and initializes tables. + """Creates an iterator, and initializes tables. - This is useful in cases where make_one_shot_iterator wouldn't work because - the graph contains a hash table that needs to be initialized. + This is useful in cases where make_one_shot_iterator wouldn't work because + the graph contains a hash table that needs to be initialized. - Args: - dataset: A `tf.data.Dataset` object. + Args: + dataset: A `tf.data.Dataset` object. - Returns: - A `tf.data.Iterator`. - """ - iterator = dataset.make_initializable_iterator() - tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) - return iterator + Returns: + A `tf.data.Iterator`. + """ + iterator = tf.data.make_initializable_iterator(dataset) + tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) + return iterator def read_dataset(file_read_func, decode_func, input_files, config): - """Reads a dataset, and handles repetition and shuffling. - - Args: - file_read_func: Function to use in tf.data.Dataset.interleave, to read - every individual file into a tf.data.Dataset. - decode_func: Function to apply to all records. - input_files: A list of file paths to read. - config: A input_reader_builder.InputReader object. - - Returns: - A tf.data.Dataset based on config. - """ - # Shard, shuffle, and read files. - filenames = tf.gfile.Glob(input_files) - num_readers = config.num_readers - if num_readers > len(filenames): - num_readers = len(filenames) - tf.logging.warning('num_readers has been reduced to %d to match input file ' - 'shards.' % num_readers) - filename_dataset = tf.data.Dataset.from_tensor_slices(tf.unstack(filenames)) - if config.shuffle: - filename_dataset = filename_dataset.shuffle( - config.filenames_shuffle_buffer_size) - elif num_readers > 1: - tf.logging.warning('`shuffle` is false, but the input data stream is ' - 'still slightly shuffled since `num_readers` > 1.') - - filename_dataset = filename_dataset.repeat(config.num_epochs or None) - - records_dataset = filename_dataset.apply( - tf.contrib.data.parallel_interleave( - file_read_func, - cycle_length=num_readers, - block_length=config.read_block_length, - sloppy=config.shuffle)) - if config.shuffle: - records_dataset = records_dataset.shuffle(config.shuffle_buffer_size) - tensor_dataset = records_dataset.map( - decode_func, num_parallel_calls=config.num_parallel_map_calls) - return tensor_dataset.prefetch(config.prefetch_size) + """Reads a dataset, and handles repetition and shuffling. + + Args: + file_read_func: Function to use in tf.data.Dataset.interleave, to read + every individual file into a tf.data.Dataset. + decode_func: Function to apply to all records. + input_files: A list of file paths to read. + config: A input_reader_builder.InputReader object. + + Returns: + A tf.data.Dataset based on config. + """ + # Shard, shuffle, and read files. + filenames = tf.gfile.Glob(input_files) + num_readers = config.num_readers + if num_readers > len(filenames): + num_readers = len(filenames) + tf.logging.warning( + f"num_readers has been reduced to {num_readers} to match input file shards." + ) + filename_dataset = tf.data.Dataset.from_tensor_slices(tf.unstack(filenames)) + if config.shuffle: + filename_dataset = filename_dataset.shuffle( + config.filenames_shuffle_buffer_size + ) + elif num_readers > 1: + tf.logging.warning( + "`shuffle` is false, but the input data stream is " + "still slightly shuffled since `num_readers` > 1." + ) + + filename_dataset = filename_dataset.repeat(config.num_epochs or None) + + if not filenames: + raise ValueError("No input files matched") + records_dataset = filename_dataset.interleave( + file_read_func, + cycle_length=num_readers, + block_length=config.read_block_length, + num_parallel_calls=num_readers, + deterministic=not config.shuffle, + ) + if config.shuffle: + records_dataset = records_dataset.shuffle(config.shuffle_buffer_size) + tensor_dataset = records_dataset.map( + decode_func, num_parallel_calls=config.num_parallel_map_calls + ) + return tensor_dataset.prefetch(config.prefetch_size) diff --git a/imagedt/tensorflow/tools/tfmodel_wrapper.py b/imagedt/tensorflow/tools/tfmodel_wrapper.py index b54042d..322fee9 100644 --- a/imagedt/tensorflow/tools/tfmodel_wrapper.py +++ b/imagedt/tensorflow/tools/tfmodel_wrapper.py @@ -1,81 +1,119 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -import tensorflow as tf - - -class TFmodel_Wrapper(object): - """docstring for TFmodel_Wrapper""" - def __init__(self, pbmodel_path, input_nodename='input', output_nodename='softmax', gpu_id=0, graph_name='classify'): - super(TFmodel_Wrapper, self).__init__() - self.graph_name = graph_name - self.pbmodel_path = pbmodel_path - self.input_node = input_nodename - self.output_node = output_nodename.split(',') - self.gpu_id = gpu_id - self._load_model() - self.reset_node_name() - self._check_node_name() - self._set_output_node() - - def _load_model(self): - # easy way! load as default graph - print("load tfmodel {0}".format(self.pbmodel_path)) - # with tf.device('/gpu:'+str(self.gpu_id)): - with tf.gfile.GFile(self.pbmodel_path, 'rb') as fid: - graph_def = tf.GraphDef() - graph_def.ParseFromString(fid.read()) - - # GPU使用率 - self.config = tf.ConfigProto() # device_count={'GPU': self.gpu_id} only gpu - # config.gpu_options.visible_device_list= str(self.gpu_id) - self.config.gpu_options.per_process_gpu_memory_fraction = 0.2 # 固定比例 - self.config.gpu_options.allow_growth = True - - self.graph = tf.get_default_graph() - tf.import_graph_def(graph_def, name=self.graph_name) - self.sess = tf.Session(graph=self.graph, config=self.config) - - def _check_node_name(self): - # checking input and output node name, notice, notice node name error - self.ops = tf.get_default_graph().get_operations() - all_tensor_names = [output.name for op in self.ops for output in op.outputs] - for node_name in ([self.input_node]+self.output_node): - if not isinstance(node_name, str): - raise ValueError("Node name: {0} should be string, like: 'input' 'data' 'softmax'.".format(node_name)) - if node_name not in all_tensor_names: - print("##### Print some node name : {0}~{1}. #####".format(all_tensor_names[:2], all_tensor_names[-2:])) - raise ValueError("node_name: {0} not in graph, please check input or output node name!".format(node_name)) - - def reset_node_name(self): - self.input_node = self.graph_name+'/'+self.input_node+':0' - for index, output_node in enumerate(self.output_node): - self.output_node[index] = self.graph_name+'/'+output_node+':0' - - def _set_output_node(self): - # Get handles to input and output tensors - self.tensor_dict = {} - intput_tensor = self.input_node - self.image_tensor = self.graph.get_tensor_by_name(intput_tensor) - for index, output_node in enumerate(self.output_node): - self.tensor_dict[output_node] = self.graph.get_tensor_by_name(output_node) - - # from imagedt.decorator import time_cost - # @time_cost - def predict(self, images): - # Run inference - output_infos = self.sess.run(self.tensor_dict, feed_dict={self.image_tensor: images})[self.output_node[0]] - infos = [] # get top 1 class and confidence - for index, info in enumerate(output_infos): - predict_cls = info.argsort()[::-1][0] - conf = info[predict_cls] - infos.append({'class':predict_cls, 'confidence': conf}) - return infos - - # from imagedt.decorator import time_cost - # @time_cost - def extract(self, image): - features = [] - output_infos = self.sess.run(self.tensor_dict, feed_dict={self.image_tensor: image})[self.output_node[1]] - return output_infos +from imagedt._backends import tensorflow_v1 + +tf = tensorflow_v1() + + +class TFmodel_Wrapper: + """docstring for TFmodel_Wrapper""" + + def __init__( + self, + pbmodel_path, + input_nodename="input", + output_nodename="softmax", + gpu_id=0, + graph_name="classify", + ): + super().__init__() + self.graph_name = graph_name + self.pbmodel_path = pbmodel_path + self.input_node = input_nodename + self.output_node = output_nodename.split(",") + self.gpu_id = gpu_id + self._load_model() + try: + self.reset_node_name() + self._check_node_name() + self._set_output_node() + except Exception: + self.close() + raise + + def _load_model(self): + # easy way! load as default graph + print(f"load tfmodel {self.pbmodel_path}") + # with tf.device('/gpu:'+str(self.gpu_id)): + with tf.gfile.GFile(self.pbmodel_path, "rb") as fid: + graph_def = tf.GraphDef() + graph_def.ParseFromString(fid.read()) + + # GPU使用率 + self.config = tf.ConfigProto() # device_count={'GPU': self.gpu_id} only gpu + # config.gpu_options.visible_device_list= str(self.gpu_id) + self.config.gpu_options.per_process_gpu_memory_fraction = 0.2 # 固定比例 + self.config.gpu_options.allow_growth = True + + self.graph = tf.Graph() + with self.graph.as_default(): + tf.import_graph_def(graph_def, name=self.graph_name) + self.sess = tf.Session(graph=self.graph, config=self.config) + + def _check_node_name(self): + # checking input and output node name, notice, notice node name error + self.ops = self.graph.get_operations() + all_tensor_names = [output.name for op in self.ops for output in op.outputs] + for node_name in [self.input_node] + self.output_node: + if not isinstance(node_name, str): + raise ValueError( + f"Node name: {node_name} should be string, like: 'input' 'data' 'softmax'." + ) + if node_name not in all_tensor_names: + print( + f"##### Print some node name : {all_tensor_names[:2]}~{all_tensor_names[-2:]}. #####" + ) + raise ValueError( + f"node_name: {node_name} not in graph, please check input or output node name!" + ) + + def reset_node_name(self): + def normalize(name): + if not isinstance(name, str): + raise ValueError("Node names must be strings") + prefix = self.graph_name + "/" if self.graph_name else "" + if not name.startswith(prefix): + name = prefix + name + if ":" not in name: + name += ":0" + return name + + self.input_node = normalize(self.input_node) + self.output_node = [normalize(name) for name in self.output_node] + + def _set_output_node(self): + # Get handles to input and output tensors + self.tensor_dict = {} + intput_tensor = self.input_node + self.image_tensor = self.graph.get_tensor_by_name(intput_tensor) + for output_node in self.output_node: + self.tensor_dict[output_node] = self.graph.get_tensor_by_name(output_node) + + # from imagedt.decorator import time_cost + # @time_cost + def predict(self, images): + # Run inference + output_infos = self.sess.run( + self.tensor_dict, feed_dict={self.image_tensor: images} + )[self.output_node[0]] + infos = [] # get top 1 class and confidence + for info in output_infos: + predict_cls = int(info.argmax()) + conf = info[predict_cls] + infos.append({"class": predict_cls, "confidence": conf}) + return infos + + # from imagedt.decorator import time_cost + # @time_cost + def extract(self, image): + output_infos = self.sess.run( + self.tensor_dict, feed_dict={self.image_tensor: image} + )[self.output_node[1]] + return output_infos + + def close(self): + self.sess.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() diff --git a/imagedt/tensorflow/tools/tfrecords.py b/imagedt/tensorflow/tools/tfrecords.py index 95eb86b..bff3c55 100644 --- a/imagedt/tensorflow/tools/tfrecords.py +++ b/imagedt/tensorflow/tools/tfrecords.py @@ -1,41 +1,18 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Preview up to two images from a TFRecord without altering global graph state.""" + +from io import BytesIO -import numpy as np -import tensorflow as tf from PIL import Image +from imagedt._backends import tensorflow_v1 -def visual_records(tfrecords_path): - # make an input queue from the tfrecord file - filename_queue = tf.train.string_input_producer([tfrecords_path]) - #创建一个reader来读取TFRecord文件 - reader = tf.TFRecordReader() - #从文件中独处一个样例。也可以使用read_up_to函数一次性读取多个样例 - _, serialized_example = reader.read(filename_queue) - #解析每一个元素。如果需要解析多个样例，可以用parse_example函数 - img_features = tf.parse_single_example( - serialized_example, - features={ - 'image/class/label': tf.FixedLenFeature([], tf.int64), - 'image/height': tf.FixedLenFeature([], tf.int64), - 'image/width': tf.FixedLenFeature([], tf.int64), - 'image/encoded': tf.FixedLenFeature([], tf.string), - }) - #tf.decode_raw可以将字符串解析成图像对应的像素数组 - # image = tf.decode_raw(img_features['image/encoded'], tf.uint8) - image = tf.image.decode_jpeg(img_features['image/encoded'], channels=3) - label = tf.cast(img_features['image/class/label'], tf.int32) - height = tf.cast(img_features['image/height'], tf.int32) - width = tf.cast(img_features['image/width'], tf.int32) - with tf.Session() as sess: - # 启动多线程 - coord = tf.train.Coordinator() - threads = tf.train.start_queue_runners(sess=sess, coord=coord) - for i in range(2): - # 获取一张图片和其对应的类型 - images, labels, height, width = sess.run([image, label, height, width]) - Image.fromarray(np.reshape(images, [height, width, 3])).show() - exit() +def visual_records(tfrecords_path): + tf = tensorflow_v1() + for index, raw in enumerate(tf.io.tf_record_iterator(str(tfrecords_path))): + if index == 2: + break + example = tf.train.Example.FromString(raw) + data = example.features.feature["image/encoded"].bytes_list.value[0] + with Image.open(BytesIO(data)) as image: + image.convert("RGB").show() diff --git a/imagedt/tensorflow/trainer.py b/imagedt/tensorflow/trainer.py index 913ef8f..e69de29 100644 --- a/imagedt/tensorflow/trainer.py +++ b/imagedt/tensorflow/trainer.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function \ No newline at end of file diff --git a/imagedt/tools/__init__.py b/imagedt/tools/__init__.py index 788af2b..9626313 100644 --- a/imagedt/tools/__init__.py +++ b/imagedt/tools/__init__.py @@ -1,9 +1,22 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from .metrics import detect_eval -from .metrics import f1_score -from .plot_curve import plot_tools -from .com_tools import * -from .text_generator import text_generator \ No newline at end of file +"""Compatibility exports, loaded on first use.""" + +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "getip": (".com_tools", "getip"), + "detect_eval": (".metrics.detect_eval", None), + "f1_score": (".metrics.f1_score", None), + "plot_tools": (".plot_curve", "plot_tools"), + "text_generator": (".text_generator.text_generator", None), + "pdf": (".pdf", None), + "metrics": (".metrics", None), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/tools/com_tools.py b/imagedt/tools/com_tools.py index a4ff2d0..9918b56 100644 --- a/imagedt/tools/com_tools.py +++ b/imagedt/tools/com_tools.py @@ -1,17 +1,13 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Small network utilities.""" import socket def getip(): try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(('www.baidu.com', 0)) - ip = s.getsockname()[0] - except: - ip = "x.x.x.x" - finally: - s.close() - return ip \ No newline at end of file + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.settimeout(2) + sock.connect(("www.baidu.com", 0)) + return sock.getsockname()[0] + except OSError: + return "x.x.x.x" diff --git a/imagedt/tools/metrics/__init__.py b/imagedt/tools/metrics/__init__.py index 6bccab7..60ca0b7 100644 --- a/imagedt/tools/metrics/__init__.py +++ b/imagedt/tools/metrics/__init__.py @@ -1,7 +1,25 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Compatibility exports, loaded on first use.""" -from .detect_eval import * -from .f1_score import * -from .roc_curve import * \ No newline at end of file +from imagedt._lazy import resolve as _resolve + +_EXPORTS = { + "parse_rec": (".detect_eval", "parse_rec"), + "parse_detect": (".detect_eval", "parse_detect"), + "load_detect_lines": (".detect_eval", "load_detect_lines"), + "get_xmls_basename": (".detect_eval", "get_xmls_basename"), + "voc_eval": (".detect_eval", "voc_eval"), + "load_label_Dict": (".f1_score", "load_label_Dict"), + "write_predict_details": (".f1_score", "write_predict_details"), + "gernerate_infos_dict": (".f1_score", "gernerate_infos_dict"), + "calculate_f1_score": (".f1_score", "calculate_f1_score"), + "roc": (".roc_curve", "roc"), +} +__all__ = list(_EXPORTS) + + +def __getattr__(name): + return _resolve(name, _EXPORTS, globals(), __name__) + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/imagedt/tools/metrics/detect_eval.py b/imagedt/tools/metrics/detect_eval.py index e02342c..42b9ef1 100644 --- a/imagedt/tools/metrics/detect_eval.py +++ b/imagedt/tools/metrics/detect_eval.py @@ -1,261 +1,173 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import os -import cPickle -import numpy as np import xml.etree.ElementTree as ET -from collections import OrderedDict -from imagedt.file import write_txt +import numpy as np + from imagedt.dir.dir_loop import loop def parse_rec(filename): - """ Parse a PASCAL VOC xml file """ + """Parse a PASCAL VOC xml file""" tree = ET.parse(filename) objects = {} - for obj in tree.findall('object'): + for obj in tree.findall("object"): obj_struct = {} - name = obj.find('name').text + name = obj.find("name").text if name not in objects: objects[name] = [] # obj_struct['pose'] = obj.find('pose').text # obj_struct['truncated'] = int(obj.find('truncated').text) - obj_struct['name'] = name - obj_struct['difficult'] = 0 - - obj_struct['confidence'] = obj.find('confidence').text if obj.find('confidence') is not None else 0.99 - bbox = obj.find('bndbox') - obj_struct['bbox'] = [float(bbox.find('xmin').text), - float(bbox.find('ymin').text), - float(bbox.find('xmax').text), - float(bbox.find('ymax').text)] - objects[obj_struct['name']].append(obj_struct) + obj_struct["name"] = name + obj_struct["difficult"] = int(obj.findtext("difficult", "0")) + + obj_struct["confidence"] = ( + obj.find("confidence").text if obj.find("confidence") is not None else 0.99 + ) + bbox = obj.find("bndbox") + obj_struct["bbox"] = [ + float(bbox.find("xmin").text), + float(bbox.find("ymin").text), + float(bbox.find("xmax").text), + float(bbox.find("ymax").text), + ] + objects[obj_struct["name"]].append(obj_struct) return objects def parse_detect(detannos): - xmls_path = loop(detannos, ['.xml']) - print('########## Reading {0} annotation files of DT.##########'.format(len(xmls_path))) + xmls_path = loop(detannos, [".xml"]) + print(f"########## Reading {len(xmls_path)} annotation files of DT.##########") infos = [] for xml_path in xmls_path: info = parse_rec(xml_path) info = [item for cls_key in info for item in info[cls_key]] basename = os.path.basename(xml_path) - infos.append([os.path.splitext(basename)[0], - [[item['confidence']]+[item['name']]+item['bbox'] for item in info]]) + infos.append( + [ + os.path.splitext(basename)[0], + [[item["confidence"]] + [item["name"]] + item["bbox"] for item in info], + ] + ) return [[item[0], item[1]] for item in infos] def load_detect_lines(detannos, conf): - txts_path = loop(detannos, ['.txt']) - print('########## Reading {0} annotation files of DT.##########'.format(len(txts_path))) + txts_path = loop(detannos, [".txt"]) + print(f"########## Reading {len(txts_path)} annotation files of DT.##########") infos = [] for txt_path in txts_path: - with open(txt_path, 'r') as f: + with open(txt_path) as f: lines = f.readlines() - true_class = os.path.basename(txt_path).split('.')[0].split('_')[-1] + true_class = os.path.basename(txt_path).split(".")[0].split("_")[-1] # 图片名[无后缀] 置信度 x1 y1 x2 y2 # target: image_name, conf, det_class, [xmin, ymin, xmax, ymax] for line in lines: - line = line.strip().split(' ') + line = line.strip().split() if float(line[1]) < conf: continue - infos.append(line[0:2]+[true_class]+line[2:]) + infos.append(line[0:2] + [true_class] + line[2:]) return infos -def get_xmls_basename(xml_path): - return [os.path.splitext(os.path.basename(item))[0] for item in loop(xml_path, ['.xml'])] - - -def voc_eval(detpath, annopath, ovthresh=0.5, det_file_type='xml', conf=0.5, set_gt_labels=None): - ################### read detect results################## - xmlnames = get_xmls_basename(annopath) - # load gt - recs = {} - for i, imagename in enumerate(xmlnames): - recs[imagename] = parse_rec(os.path.join(annopath, '{0}.xml').format(imagename)) - print('########## Reading {0} annotation files of GT.##########'.format(len(xmlnames))) - - # extract gt objects - gt_class_bbox = {} - class_recs = {} - npos = 0 - for imagename in xmlnames: - if det_file_type == 'xml': - R = [obj for objs in recs[imagename].values() for obj in objs] - #print(len(R)) - else: - R = [obj for objs in recs[imagename].values() for obj in objs if int(obj['name']) !=0] - bbox = np.array([x['bbox'] for x in R]) - difficult = np.array([x['difficult'] for x in R]).astype(np.bool) - classes = np.array([x['name'] for x in R]) - for x in R: - label = x['name'] - if label not in gt_class_bbox.keys(): - gt_class_bbox[label] = 1 - else: - gt_class_bbox[label] += 1 - det = [False] * len(R) - npos = npos + sum(~difficult) - class_recs[imagename] = {'bbox': bbox, - 'difficult': difficult, - 'det': det, - 'classes':classes} - print("Ground Truth bbox num: {}\nsum: {}".format(gt_class_bbox, sum(gt_class_bbox.values()))) - - ################### read detect results ################## - if det_file_type == 'xml': - lines = parse_detect(detpath) - # splitlines : image_name, conf, det_class, [xmin, ymin, xmax, ymax] - #splitlines = [[item[0]]+x for item in lines for x in item[1]] - splitlines = [] - for item in lines: - for x in item[1]: - #print(x) - #print("conf: {}\tthresh: {}".format(x[0], conf)) - if float(x[0]) < conf: - #print(x) - x[1] = '0' - #print(x) - splitlines.append([item[0]]+x) - else: - splitlines = load_detect_lines(detpath, conf=conf) - - image_ids = [x[0] for x in splitlines] - confidence = np.array([float(x[1]) for x in splitlines]) - det_classes = np.array([x[2] for x in splitlines]) - # - det_class_bbox = {} - for x in splitlines: - label = x[2] - if label not in det_class_bbox.keys(): - det_class_bbox[label] = 1 - else: - det_class_bbox[label] += 1 - print("Predict bbox num: {}\nsum: {}".format(det_class_bbox,sum(det_class_bbox.values()))) - # - BB = np.array([[float(z) for z in x[3:]] for x in splitlines]) - - # sort by confidence - sorted_ind = np.argsort(-confidence) - # sorted_scores = np.sort(-confidence) - BB = BB[sorted_ind] - det_classes = det_classes[sorted_ind] - image_ids = [image_ids[x] for x in sorted_ind] - # go down dets and mark TPs and FPs - nd = len(image_ids) - tp = np.zeros(nd) - fp = np.zeros(nd) - - cls_tp = np.zeros(nd) - cls_fp = np.zeros(nd) - other_tp = 0 - - if set_gt_labels is None: - gt_labels = ['9265','9304','9320','9282','9334','9273','9252','9294'] +def get_xmls_basename(xml_path): + return [ + os.path.splitext(os.path.basename(item))[0] for item in loop(xml_path, [".xml"]) + ] + + +def voc_eval( + detpath, annopath, ovthresh=0.5, det_file_type="xml", conf=0.5, set_gt_labels=None +): + """Macro F1 with class-aware, confidence-ordered VOC box matching. + + VOC coordinates are inclusive; an IoU strictly greater than ovthresh matches. + Difficult ground truths are ignored. Empty evaluated label sets score zero. + """ + if det_file_type not in ("xml", "txt"): + raise ValueError("det_file_type must be xml or txt") + if not 0 <= ovthresh <= 1 or not 0 <= conf <= 1: + raise ValueError("thresholds must be between zero and one") + truth = {} + labels = set() + for path in loop(annopath, [".xml"]): + key = os.path.splitext(os.path.basename(path))[0] + if key in truth: + raise ValueError(f"Duplicate ground-truth image id: {key}") + truth[key] = [obj for group in parse_rec(path).values() for obj in group] + labels.update(obj["name"] for obj in truth[key]) + if det_file_type == "xml": + detections = [ + [key] + obj + for key, objects in parse_detect(detpath) + for obj in objects + if float(obj[0]) >= conf + ] else: - gt_labels = set([ite for item in class_recs.values() for ite in item['classes']]) - - tp = {} - for label in gt_labels: - if label not in tp.keys(): - tp[label]=0 - fp = {} - for label in gt_labels: - if label not in fp.keys(): - fp[label]=0 - tn = {} - for label in gt_labels: - if label not in tn.keys(): - tn[label]=0 - fn = {} - for label in gt_labels: - if label not in fn.keys(): - fn[label]=0 - - for d in range(nd): - R = class_recs[image_ids[d]] - bb = BB[d, :].astype(float) - ovmax = -np.inf - BBGT = R['bbox'].astype(float) - - if BBGT.size > 0: - # compute overlaps - # intersection - ixmin = np.maximum(BBGT[:, 0], bb[0]) - iymin = np.maximum(BBGT[:, 1], bb[1]) - ixmax = np.minimum(BBGT[:, 2], bb[2]) - iymax = np.minimum(BBGT[:, 3], bb[3]) - iw = np.maximum(ixmax - ixmin + 1., 0.) - ih = np.maximum(iymax - iymin + 1., 0.) - inters = iw * ih - - # union - uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + - (BBGT[:, 2] - BBGT[:, 0] + 1.) * - (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) - - overlaps = inters / uni - ovmax = np.max(overlaps) - jmax = np.argmax(overlaps) - - if ovmax > ovthresh: - if not R['det'][jmax]: - R['det'][jmax] = 1 - if int(det_classes[d]) == int(R['classes'][jmax]): - for label in gt_labels: - if label == det_classes[d]: - tp[label] += 1 - else: - tn[label] += 1 - else: - for label in gt_labels: - if label == det_classes[d]: - fp[label] += 1 - elif label == R['classes'][jmax]: - fn[label] += 1 - else: - tn[label] += 1 - else: - for label in gt_labels: - #if label != R['classes'][jmax]: - if label != det_classes[d]: - tn[label] += 1 - else: - fp[label] += 1 - else: - for label in gt_labels: - if label != R['classes'][jmax]: - tn[label] += 1 - else: - fn[label] += 1 - - for label in gt_labels: - miss_gt = gt_class_bbox[label] - (tp[label] + fn[label]) - if miss_gt > 0: - fn[label] += miss_gt - - print("ground truth: {0}, detect results {1}".format(npos, nd)) - precision = {} - recall = {} - f1_score = {} - for label in gt_labels: - if label == '0': + detections = load_detect_lines(detpath, conf) + labels.update(str(row[2]) for row in detections) + labels = set(map(str, set_gt_labels)) if set_gt_labels is not None else labels + labels.discard("0") + tp = dict.fromkeys(labels, 0) + fp = dict.fromkeys(labels, 0) + totals = { + label: sum( + obj["name"] == label and not obj["difficult"] + for objects in truth.values() + for obj in objects + ) + for label in labels + } + matched = set() + for row in sorted(detections, key=lambda row: -float(row[1])): + image, _, label, *coords = row + label = str(label) + if label not in labels: continue - print("Label: {}".format(label)) - print("\ttp: {}\tfp: {}\ttn: {}\tfn: {}\ttotal: {}".format(tp[label],fp[label],tn[label],fn[label], sum([tp[label],fp[label],tn[label],fn[label]]))) - precision[label] = tp[label] * 1.0 / np.maximum( (tp[label] + fp[label]), np.finfo(np.float64).eps) - recall[label] = tp[label] * 1.0 / np.maximum( (tp[label] + fn[label]), np.finfo(np.float64).eps) - f1_score[label] = round(2*(precision[label]*recall[label])/np.maximum((precision[label]+recall[label]), np.finfo(np.float64).eps), 4) - print ("\t#### recall: {0}, precision: {1} ####".format(round(recall[label], 4), round(precision[label], 4))) - print ("\t#### f1-score: {0} ##########".format(f1_score[label])) - - mean_f1 = np.mean(f1_score.values()) - print("Mean F1: {}\n".format(round(mean_f1, 4))) + box = np.asarray(coords, dtype=float) + if box.shape != (4,) or np.any(box[2:] < box[:2]): + raise ValueError("invalid detection box") + candidates = [ + (i, obj) + for i, obj in enumerate(truth.get(image, [])) + if obj["name"] == label + ] + if not candidates: + fp[label] += 1 + continue + boxes = np.asarray([obj["bbox"] for _, obj in candidates], dtype=float) + intersection = np.prod( + np.maximum( + np.minimum(boxes[:, 2:], box[2:]) + - np.maximum(boxes[:, :2], box[:2]) + + 1, + 0, + ), + axis=1, + ) + union = ( + np.prod(box[2:] - box[:2] + 1) + + np.prod(boxes[:, 2:] - boxes[:, :2] + 1, axis=1) + - intersection + ) + overlap = np.divide( + intersection, union, out=np.zeros_like(intersection), where=union > 0 + ) + best = int(np.argmax(overlap)) + index, obj = candidates[best] + if overlap[best] > ovthresh: + if obj["difficult"]: + continue + if (image, index) not in matched: + matched.add((image, index)) + tp[label] += 1 + continue + fp[label] += 1 + scores = [ + 2 * tp[label] / max(totals[label] + tp[label] + fp[label], 1) + for label in sorted(labels) + ] + mean_f1 = float(np.mean(scores)) if scores else 0.0 + print(f"Mean F1: {mean_f1:.4f}") return mean_f1 diff --git a/imagedt/tools/metrics/f1_score.py b/imagedt/tools/metrics/f1_score.py index 7494704..72419bf 100644 --- a/imagedt/tools/metrics/f1_score.py +++ b/imagedt/tools/metrics/f1_score.py @@ -1,157 +1,221 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - import os -import numpy as np from collections import OrderedDict +import numpy as np + import imagedt + def load_label_Dict(lines): - # line: onject_id, class_id, sku_name - return dict([(line.split('\t')[1], line.split('\t')) for line in lines]) + # line: onject_id, class_id, sku_name + return dict([(line.split("\t")[1], line.split("\t")) for line in lines]) def write_predict_details(predict_infos, label_infos, label_map_dict): - # save test image details, init list - jpg_details = [['Image_name', 'True_class_name', 'Label_id', 'Predict_id', 'Confidence', 'Match']] - for ind in range(len(predict_infos)): - # GT infos - true_class = label_infos[ind].split(',')[1] - true_objectId, _, true_class_name = label_map_dict[true_class] - # test infos - file_name, pre_class, pred_conf = predict_infos[ind].split(',') - pred_object_id, _, _ = label_map_dict[pre_class] - # set key infos - match = 1 if int(true_class) == int(pre_class) else 0 - file_name = os.path.basename(file_name) - jpg_details.append([file_name, true_class_name, true_objectId, pred_object_id, pred_conf, match]) - return jpg_details + # save test image details, init list + jpg_details = [ + [ + "Image_name", + "True_class_name", + "Label_id", + "Predict_id", + "Confidence", + "Match", + ] + ] + for ind in range(len(predict_infos)): + # GT infos + true_class = label_infos[ind].split(",")[1] + true_objectId, _, true_class_name = label_map_dict[true_class] + # test infos + file_name, pre_class, pred_conf = predict_infos[ind].split(",") + pred_object_id, _, _ = label_map_dict[pre_class] + # set key infos + match = 1 if int(true_class) == int(pre_class) else 0 + file_name = os.path.basename(file_name) + jpg_details.append( + [ + file_name, + true_class_name, + true_objectId, + pred_object_id, + pred_conf, + match, + ] + ) + return jpg_details def _total_f1_score(cls_dict): - cor = sum([cls_dict[cls_id]['correct'] for cls_id in cls_dict]) - sum_count = sum([cls_dict[cls_id]['cls_pre_count'] for cls_id in cls_dict]) - total_precision = round(float(cor)/sum_count, 4) + cor = sum([cls_dict[cls_id]["correct"] for cls_id in cls_dict]) + sum_count = sum([cls_dict[cls_id]["cls_pre_count"] for cls_id in cls_dict]) + total_precision = round(float(cor) / max(sum_count, 1), 4) - tp_count = sum([cls_dict[cls_id]['TP'] for cls_id in cls_dict]) - sum_count = sum([cls_dict[cls_id]['count'] for cls_id in cls_dict]) - total_recall = round(float(cor)/sum_count, 4) - return total_recall, total_precision + sum([cls_dict[cls_id]["TP"] for cls_id in cls_dict]) + sum_count = sum([cls_dict[cls_id]["count"] for cls_id in cls_dict]) + total_recall = round(float(cor) / max(sum_count, 1), 4) + return total_recall, total_precision def _f1_score(cls_dict, label_map_dict): - #index_item = map(int, set([item[0] for item in label_map_dict.values() if int(item[0]) != -1])) - index_item = set([item[0] for item in label_map_dict.values()]) - results = [['SKU_name', 'Object_id', 'Sample_count','TP', 'FN', 'FP', 'Recall', 'Precision', 'F1-score']] - for cls_id in cls_dict: - object_id = cls_dict[cls_id]['object_id'] - count = cls_dict[cls_id]['count'] - TP = cls_dict[cls_id]['TP'] - FN = cls_dict[cls_id]['FN'] - FP = cls_dict[cls_id]['FP'] - - cls_dict[cls_id]['cls_pre_count'] = TP+FP - - recall = round(float(TP)/np.maximum((TP+FN), np.finfo(np.float64).eps), 4) - precision = round(float(TP)/np.maximum((TP+FP), np.finfo(np.float64).eps), 4) - f1_score = round(2*recall*precision/np.maximum((recall+precision), np.finfo(np.float64).eps), 4) - - # col_list = ['SKU_name', 'Object_id', 'Sample_count','TP', 'FN', 'FP', 'Recall', 'Precision'] - sku_name = label_map_dict[cls_id][2] - if sku_name.startswith('other'): - continue - value_list = [sku_name, object_id, count, TP, FN, FP, recall, precision, f1_score] - results.append(value_list) - - results = sorted(results, key=lambda x: x[-1])[::-1] - return results + # index_item = map(int, set([item[0] for item in label_map_dict.values() if int(item[0]) != -1])) + set([item[0] for item in list(label_map_dict.values())]) + results = [ + [ + "SKU_name", + "Object_id", + "Sample_count", + "TP", + "FN", + "FP", + "Recall", + "Precision", + "F1-score", + ] + ] + for cls_id in cls_dict: + object_id = cls_dict[cls_id]["object_id"] + count = cls_dict[cls_id]["count"] + TP = cls_dict[cls_id]["TP"] + FN = cls_dict[cls_id]["FN"] + FP = cls_dict[cls_id]["FP"] + + cls_dict[cls_id]["cls_pre_count"] = TP + FP + + recall = round(float(TP) / np.maximum((TP + FN), np.finfo(np.float64).eps), 4) + precision = round( + float(TP) / np.maximum((TP + FP), np.finfo(np.float64).eps), 4 + ) + f1_score = round( + 2 + * recall + * precision + / np.maximum((recall + precision), np.finfo(np.float64).eps), + 4, + ) + + # col_list = ['SKU_name', 'Object_id', 'Sample_count','TP', 'FN', 'FP', 'Recall', 'Precision'] + sku_name = label_map_dict[cls_id][2] + if sku_name.startswith("other"): + continue + value_list = [ + sku_name, + object_id, + count, + TP, + FN, + FP, + recall, + precision, + f1_score, + ] + results.append(value_list) + + results = results[:1] + sorted(results[1:], key=lambda x: x[-1], reverse=True) + return results def gernerate_infos_dict(predicts, true_labels, label_map_dict): - # predict infos data struct - cls_dict = OrderedDict() - - for ind in range(len(predicts)): - true_class = true_labels[ind].split(',')[1] - true_class_name = label_map_dict[true_class][2] - true_objectId = label_map_dict[true_class][0] - - file_name, pre_class, pred_conf = predicts[ind].split(',') - pred_cls_name = label_map_dict[pre_class][2] - pred_object_id = label_map_dict[pre_class][0] - - if true_class not in cls_dict: - cls_dict[true_class] = {} - cls_dict[true_class]['TP'] = 0 - cls_dict[true_class]['FN'] = 0 - cls_dict[true_class]['FP'] = 0 - cls_dict[true_class]['count'] = 0 - cls_dict[true_class]['object_id'] = 0 - cls_dict[true_class]['precision'] = 0 - cls_dict[true_class]['recall'] = 0 - cls_dict[true_class]['correct'] = 0 - cls_dict[true_class]['cls_pre_count'] = 0 - - if pre_class not in cls_dict: - cls_dict[pre_class] = {} - cls_dict[pre_class]['TP'] = 0 - cls_dict[pre_class]['FN'] = 0 - cls_dict[pre_class]['FP'] = 0 - cls_dict[pre_class]['count'] = 0 - cls_dict[pre_class]['object_id'] = 0 - cls_dict[pre_class]['precision'] = 0 - cls_dict[pre_class]['recall'] = 0 - cls_dict[pre_class]['correct'] = 0 - cls_dict[pre_class]['cls_pre_count'] = 0 - - if int(pre_class) == int(true_class): - cls_dict[true_class]['TP'] += 1 - cls_dict[true_class]['correct'] += 1 - cls_dict[true_class]['object_id'] = true_objectId - else: - cls_dict[true_class]['FN'] += 1 - cls_dict[pre_class]['FP'] += 1 - cls_dict[pre_class]['object_id'] = pred_object_id - - cls_dict[true_class]['count'] += 1 - return cls_dict + if len(predicts) != len(true_labels): + raise ValueError("prediction and ground-truth lengths differ") + # predict infos data struct + cls_dict = OrderedDict() + + for ind in range(len(predicts)): + true_class = true_labels[ind].split(",")[1] + label_map_dict[true_class][2] + true_objectId = label_map_dict[true_class][0] + + file_name, pre_class, pred_conf = predicts[ind].split(",") + label_map_dict[pre_class][2] + pred_object_id = label_map_dict[pre_class][0] + + if true_class not in cls_dict: + cls_dict[true_class] = {} + cls_dict[true_class]["TP"] = 0 + cls_dict[true_class]["FN"] = 0 + cls_dict[true_class]["FP"] = 0 + cls_dict[true_class]["count"] = 0 + cls_dict[true_class]["object_id"] = true_objectId + cls_dict[true_class]["precision"] = 0 + cls_dict[true_class]["recall"] = 0 + cls_dict[true_class]["correct"] = 0 + cls_dict[true_class]["cls_pre_count"] = 0 + + if pre_class not in cls_dict: + cls_dict[pre_class] = {} + cls_dict[pre_class]["TP"] = 0 + cls_dict[pre_class]["FN"] = 0 + cls_dict[pre_class]["FP"] = 0 + cls_dict[pre_class]["count"] = 0 + cls_dict[pre_class]["object_id"] = pred_object_id + cls_dict[pre_class]["precision"] = 0 + cls_dict[pre_class]["recall"] = 0 + cls_dict[pre_class]["correct"] = 0 + cls_dict[pre_class]["cls_pre_count"] = 0 + + if int(pre_class) == int(true_class): + cls_dict[true_class]["TP"] += 1 + cls_dict[true_class]["correct"] += 1 + cls_dict[true_class]["object_id"] = true_objectId + else: + cls_dict[true_class]["FN"] += 1 + cls_dict[pre_class]["FP"] += 1 + cls_dict[pre_class]["object_id"] = pred_object_id + + cls_dict[true_class]["count"] += 1 + return cls_dict def calculate_f1_score(predict_file, gt_file, labelt_file): - """ - predict_file one line: filename, predict_cls, predict_conf] - gt_file one line: filename, label_cls - labelt_file one line: object_id, cls_id, sku_name - """ - labelt_lines = imagedt.file.readlines(labelt_file) - predict_lines = imagedt.file.readlines(predict_file) - true_lines = imagedt.file.readlines(gt_file) - - try: - assert len(predict_lines) == len(true_lines) - except Exception as e: - raise ValueError("len(predict infos) != len(groundtrue infos, please check datas.") - - try: - assert '\t' in labelt_lines[0] - except Exception as e: - raise ValueError("check label_t file format, split by '/t'") - - # init classifier - label_map_dict = load_label_Dict(labelt_lines) - cls_dict = gernerate_infos_dict(predict_lines, true_lines, label_map_dict) - - # save test image infos - jpg_details = write_predict_details(predict_lines, true_lines, label_map_dict) - jpg_infos_name = os.path.join(os.path.dirname(predict_file), 'image_classify_infos.csv') - imagedt.file.write_csv(jpg_details, jpg_infos_name) - print("Saved test image infos, save dir:{0}".format(jpg_infos_name)) - - # save f1 score - f1_score_pandas = _f1_score(cls_dict, label_map_dict) - exlce_name = os.path.join(os.path.dirname(predict_file), 'image_recall_precision.csv') - imagedt.file.write_csv(f1_score_pandas, exlce_name) - print("Saved f1 score, save dir:{0}".format(exlce_name)) + """ + predict_file one line: filename, predict_cls, predict_conf] + gt_file one line: filename, label_cls + labelt_file one line: object_id, cls_id, sku_name + """ + labelt_lines = imagedt.file.readlines(labelt_file) + predict_lines = imagedt.file.readlines(predict_file) + true_lines = imagedt.file.readlines(gt_file) + + if len(predict_lines) != len(true_lines): + raise ValueError( + "len(predict infos) != len(groundtrue infos, please check datas." + ) + + if not labelt_lines or "\t" not in labelt_lines[0]: + raise ValueError("check label_t file format, split by '/t'") + + # Align by basename rather than trusting file row order. + def by_name(lines): + result = {} + for line in lines: + key = os.path.basename(line.split(",")[0]) + if key in result: + raise ValueError(f"duplicate image name: {key}") + result[key] = line + return result + + predictions = by_name(predict_lines) + truth = by_name(true_lines) + if predictions.keys() != truth.keys(): + raise ValueError("prediction and ground-truth image names differ") + true_lines = [truth[key] for key in predictions] + # init classifier + label_map_dict = load_label_Dict(labelt_lines) + cls_dict = gernerate_infos_dict(predict_lines, true_lines, label_map_dict) + + # save test image infos + jpg_details = write_predict_details(predict_lines, true_lines, label_map_dict) + jpg_infos_name = os.path.join( + os.path.dirname(predict_file), "image_classify_infos.csv" + ) + imagedt.file.write_csv(jpg_details, jpg_infos_name) + print(f"Saved test image infos, save dir:{jpg_infos_name}") + + # save f1 score + f1_score_pandas = _f1_score(cls_dict, label_map_dict) + exlce_name = os.path.join( + os.path.dirname(predict_file), "image_recall_precision.csv" + ) + imagedt.file.write_csv(f1_score_pandas, exlce_name) + print(f"Saved f1 score, save dir:{exlce_name}") diff --git a/imagedt/tools/metrics/roc_curve.py b/imagedt/tools/metrics/roc_curve.py index 03ed9bb..8b0d5fc 100644 --- a/imagedt/tools/metrics/roc_curve.py +++ b/imagedt/tools/metrics/roc_curve.py @@ -1,65 +1,63 @@ -# coding: utf-8 import os -import numpy as np + import matplotlib.pyplot as plt +import numpy as np import imagedt def roc(test_file, show_fig=False): - lines = imagedt.file.readlines(test_file) + imagedt.file.readlines(test_file) - # line: Image_name,True_class_name,Label_id,Predict_id,Confidence,Match + import csv + + with open(test_file, encoding="utf-8-sig", newline="") as stream: + rows = list(csv.reader(stream)) + if not rows: + raise ValueError("ROC input is empty") try: - int(lines[0].split(',')[-1]) - start_line_num = 0 + int(rows[0][-1]) except ValueError: - start_line_num = 1 - - results = np.array([[line.split(',')[-2], line.split(',')[-1]] for line in lines[start_line_num:]]) - prob_arr = np.array(results[:, 0], dtype=float) - match_arr = np.array(results[:, 1], dtype=int) - - # imagedt.tools.set_pdb() - save_lines, thresvalue = [], [] - tpr, fnr = [], [] - for index, thres in enumerate(np.arange(0,1.001,0.001)): - tp, fn, fp, tn = 0, 0, 0, 0 - for res_ind, pre_prob in enumerate(prob_arr): - if float(prob_arr[res_ind]) >= thres: - if int(match_arr[res_ind]): - tp += 1 - else: - fp += 1 - else: - if int(match_arr[res_ind]): - fn += 1 - else: - tn += 1 - - y = float(tp) / np.maximum(float(tp + fn), np.finfo(np.float64).eps) - x = float(fp) / np.maximum(float(fp + tn), np.finfo(np.float64).eps) - thresvalue.append(thres) - tpr.append(y) - fnr.append(x) - save_lines.append([str(x), str(y), str(thres)]) + rows = rows[1:] + if not rows: + raise ValueError("ROC input has no samples") + probability = np.asarray([row[-2] for row in rows], dtype=float) + matches = np.asarray([row[-1] for row in rows], dtype=int) + if ( + np.any(~np.isfinite(probability)) + or np.any((probability < 0) | (probability > 1)) + or np.any((matches != 0) & (matches != 1)) + ): + raise ValueError("expected finite probabilities in [0, 1] and binary matches") + thresholds = np.arange(0, 1.001, 0.001) + positive = np.sort(probability[matches == 1]) + negative = np.sort(probability[matches == 0]) + # O(N log N + T log N), without a threshold-by-sample Python loop. + tpr = (len(positive) - np.searchsorted(positive, thresholds, side="left")) / max( + len(positive), 1 + ) + fnr = (len(negative) - np.searchsorted(negative, thresholds, side="left")) / max( + len(negative), 1 + ) + thresvalue = thresholds + save_lines = np.column_stack((fnr, tpr, thresholds)).tolist() save_dir = os.path.dirname(test_file) - - imagedt.file.write_csv(save_lines, os.path.join(save_dir, 'thres-fpr-tpr.csv')) - fig=plt.figure() + imagedt.file.write_csv(save_lines, os.path.join(save_dir, "thres-fpr-tpr.csv")) + fig = plt.figure() ax = fig.add_subplot(111) - ax.set_xscale('log') + ax.set_xscale("log") plt.grid() - plt.xlabel('FPR') - plt.ylabel('TPR/THRES') - plt.title('ROC-THRES') - ax.plot(fnr, tpr, color='red') - ax.plot(fnr, thresvalue, color='green') - plt.legend(('roc', 'fpr-thres'),loc='best') - - plt.savefig(os.path.join(save_dir, 'roc.png'), dpi=300) - print ("ROC curve figure save: {0}".format(save_dir)) + plt.xlabel("FPR") + plt.ylabel("TPR/THRES") + plt.title("ROC-THRES") + ax.plot(fnr, tpr, color="red") + ax.plot(fnr, thresvalue, color="green") + plt.legend(("roc", "fpr-thres"), loc="best") + + plt.savefig(os.path.join(save_dir, "roc.png"), dpi=300) + print(f"ROC curve figure save: {save_dir}") if show_fig: plt.show() + plt.close(fig) diff --git a/imagedt/tools/pdf.py b/imagedt/tools/pdf.py index 8e93a72..52a2d41 100644 --- a/imagedt/tools/pdf.py +++ b/imagedt/tools/pdf.py @@ -1,13 +1,34 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function +"""Render PDF pages with a portable optional PDFium backend.""" -import os -from wand.image import Image +from pathlib import Path + +from imagedt._backends import require def pdf2image(pdf_path, save_out=None, dpi=300): - with Image(filename=pdf_path, resolution=dpi) as img: - if save_out is None: - save_out = pdf_path[:-4] + '.jpg' - img.save(filename=save_out) + if dpi <= 0: + raise ValueError("dpi must be positive") + pdfium = require("pypdfium2", "pdf") + output = ( + Path(save_out) if save_out is not None else Path(pdf_path).with_suffix(".jpg") + ) + with pdfium.PdfDocument(str(pdf_path)) as document: + for index in range(len(document)): + destination = ( + output + if len(document) == 1 + else output.with_name(f"{output.stem}-{index}{output.suffix}") + ) + page = document[index] + try: + bitmap = page.render(scale=dpi / 72) + try: + image = bitmap.to_pil() + try: + image.convert("RGB").save(destination) + finally: + image.close() + finally: + bitmap.close() + finally: + page.close() diff --git a/imagedt/tools/plot_curve.py b/imagedt/tools/plot_curve.py index f5eb3ff..14e1010 100644 --- a/imagedt/tools/plot_curve.py +++ b/imagedt/tools/plot_curve.py @@ -1,84 +1,100 @@ -#coding: utf-8 - import os -import imagedt -import numpy as np +import numpy as np from matplotlib import pyplot as plt +import imagedt -class PlotCurve(object): - """docstring for PlotCurev""" - def __init__(self): - super(PlotCurve, self).__init__() - self.params = { - 'axes.labelsize': '35', - 'xtick.labelsize':'10', - 'ytick.labelsize':'15', - 'lines.linewidth':2 , - 'legend.fontsize': '8', - 'figure.figsize': '24, 9', } - plt.rcParams.update(self.params) - - def load_dataset_infos(self, data_dir): - class_names = os.listdir(data_dir) - pos_samples_dict = {} - for class_name in class_names: - if os.path.isfile(os.path.join(data_dir, class_name)): - continue - if class_name.startswith('other') or class_name.startswith('0'): - continue - # pos_samples_dict[class_name] = len(imagedt.dir.loop(os.path.join(data_dir, class_name), ['.jpg', '.png'])) - pos_samples_dict[class_name] = len(imagedt.dir.loop(os.path.join(data_dir, class_name))) - if len(imagedt.dir.loop(os.path.join(data_dir, class_name))) < 10: - print(class_name) - return pos_samples_dict - - def set_xy_values(self, sample_dict): - pos_list = [[class_name, sample_dict[class_name]] for class_name in sample_dict] - pos_list = np.array(sorted(pos_list, key=lambda x: int(x[1]))[::-1]) - self.x = range(len(pos_list[:, 0])) - self.y = map(int, pos_list[:, 1]) - - def add_bar_values(self, rects): - for index, rect in enumerate(rects): - height = rect.get_height() - plt.text(rect.get_x() + rect.get_width() / 1.5, height, self.y[index], ha='center', va='bottom',fontsize=3) - - def add_avg_yline(self): - line_arange = np.arange(0, len(self.x), len(self.x)/5.) - y_ite = [100, 200, 300, 400, 500] - for index, ite in enumerate(line_arange): - plt.axhline(y=int(y_ite[index]), color='blue') - plt.axvline(x=int(ite), color='blue') - - def plot_datas(self, data_dir, add_bar_values=True, title=None, curve_type='bar'): - print("loading data infos......") - sample_dict = self.load_dataset_infos(data_dir) - # get x,y values - self.set_xy_values(sample_dict) - # plot bar - if curve_type == 'bar': - rects = plt.bar(range(len(self.x)), self.y, 1, align='edge', color='rgby') - if add_bar_values: - self.add_bar_values(rects) - self.add_avg_yline() - else: - plt.plot(self.x, self.y) - - plt.ylim(ymin=0, ymax=max(self.y)) - add_str = '_'+str(len(self.x))+' classes' - title = os.path.basename(data_dir)+add_str if title is None else title+add_str - plt.title(title) - - plt.xticks(range(len(self.x)), self.x, rotation=270, fontsize=3) - plt.ylabel("sample count") - - save_figure_path = os.path.join(os.path.dirname(data_dir), 'datasets.png') - print("saving figure, path {0}......".format(save_figure_path)) - plt.savefig(save_figure_path, dpi=300) +class PlotCurve: + """docstring for PlotCurev""" + + def __init__(self): + super().__init__() + self.params = { + "axes.labelsize": "35", + "xtick.labelsize": "10", + "ytick.labelsize": "15", + "lines.linewidth": 2, + "legend.fontsize": "8", + "figure.figsize": "24, 9", + } + + def load_dataset_infos(self, data_dir): + class_names = os.listdir(data_dir) + pos_samples_dict = {} + for class_name in class_names: + if os.path.isfile(os.path.join(data_dir, class_name)): + continue + if class_name.startswith("other") or class_name.startswith("0"): + continue + # pos_samples_dict[class_name] = len(imagedt.dir.loop(os.path.join(data_dir, class_name), ['.jpg', '.png'])) + pos_samples_dict[class_name] = len( + imagedt.dir.loop(os.path.join(data_dir, class_name)) + ) + if pos_samples_dict[class_name] < 10: + print(class_name) + return pos_samples_dict + + def set_xy_values(self, sample_dict): + values = sorted(sample_dict.items(), key=lambda pair: pair[1], reverse=True) + self.x = list(range(len(values))) + self.y = [int(count) for _, count in values] + + def add_bar_values(self, rects): + for index, rect in enumerate(rects): + height = rect.get_height() + plt.text( + rect.get_x() + rect.get_width() / 1.5, + height, + self.y[index], + ha="center", + va="bottom", + fontsize=3, + ) + + def add_avg_yline(self): + line_arange = np.linspace(0, len(self.x), 5, endpoint=False) + y_ite = [100, 200, 300, 400, 500] + for index, ite in enumerate(line_arange): + plt.axhline(y=int(y_ite[index]), color="blue") + plt.axvline(x=int(ite), color="blue") + + def plot_datas(self, data_dir, add_bar_values=True, title=None, curve_type="bar"): + fig = plt.figure(figsize=(12, 5)) + print("loading data infos......") + sample_dict = self.load_dataset_infos(data_dir) + # get x,y values + self.set_xy_values(sample_dict) + # plot bar + if curve_type == "bar": + rects = plt.bar( + list(range(len(self.x))), + self.y, + 1, + align="edge", + color=["r", "g", "b", "y"], + ) + if add_bar_values: + self.add_bar_values(rects) + self.add_avg_yline() + else: + plt.plot(self.x, self.y) + + plt.ylim(ymin=0, ymax=max(self.y, default=1) or 1) + add_str = "_" + str(len(self.x)) + " classes" + title = ( + os.path.basename(data_dir) + add_str if title is None else title + add_str + ) + plt.title(title) + + plt.xticks(list(range(len(self.x))), self.x, rotation=270, fontsize=3) + plt.ylabel("sample count") + + save_figure_path = os.path.join(os.path.dirname(data_dir), "datasets.png") + print(f"saving figure, path {save_figure_path}......") + plt.savefig(save_figure_path, dpi=300) + plt.close(fig) plot_tools = PlotCurve() - diff --git a/imagedt/tools/text_generator/__init__.py b/imagedt/tools/text_generator/__init__.py index eb4cf61..6260c23 100644 --- a/imagedt/tools/text_generator/__init__.py +++ b/imagedt/tools/text_generator/__init__.py @@ -1,5 +1 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from . import * \ No newline at end of file +"""Synthetic text utilities; access submodules explicitly.""" diff --git a/imagedt/tools/text_generator/background_generator.py b/imagedt/tools/text_generator/background_generator.py index 90d91ab..866304e 100644 --- a/imagedt/tools/text_generator/background_generator.py +++ b/imagedt/tools/text_generator/background_generator.py @@ -1,16 +1,17 @@ -import cv2 import math import os import random + +import cv2 import numpy as np +from PIL import Image -from PIL import Image, ImageDraw, ImageFilter -class BackgroundGenerator(object): +class BackgroundGenerator: @classmethod def gaussian_noise(cls, height, width): """ - Create a background with Gaussian noise (to mimic paper) + Create a background with Gaussian noise (to mimic paper) """ # We create an all white image @@ -19,74 +20,60 @@ def gaussian_noise(cls, height, width): # We add gaussian noise cv2.randn(image, 235, 10) - return Image.fromarray(image).convert('RGB') + return Image.fromarray(image).convert("RGB") @classmethod def plain_white(cls, height, width): """ - Create a plain white background + Create a plain white background """ - return Image.new("L", (width, height), 255).convert('RGB') + return Image.new("L", (width, height), 255).convert("RGB") @classmethod def quasicrystal(cls, height, width): """ - Create a background with quasicrystal (https://en.wikipedia.org/wiki/Quasicrystal) + Create a background with quasicrystal (https://en.wikipedia.org/wiki/Quasicrystal) """ - image = Image.new("L", (width, height)) - pixels = image.load() - - frequency = random.random() * 30 + 20 # frequency - phase = random.random() * 2 * math.pi # phase - rotation_count = random.randint(10, 20) # of rotations - - for kw in range(width): - y = float(kw) / (width - 1) * 4 * math.pi - 2 * math.pi - for kh in range(height): - x = float(kh) / (height - 1) * 4 * math.pi - 2 * math.pi - z = 0.0 - for i in range(rotation_count): - r = math.hypot(x, y) - a = math.atan2(y, x) + i * math.pi * 2.0 / rotation_count - z += math.cos(r * math.sin(a) * frequency + phase) - c = int(255 - round(255 * z / rotation_count)) - pixels[kw, kh] = c # grayscale - return image.convert('RGB') + if width < 1 or height < 1: + raise ValueError("background dimensions must be positive") + y = np.linspace(-2 * math.pi, 2 * math.pi, width) if width > 1 else np.zeros(1) + x = ( + np.linspace(-2 * math.pi, 2 * math.pi, height) + if height > 1 + else np.zeros(1) + ) + yy, xx = np.meshgrid(y, x) + frequency = random.random() * 30 + 20 + phase = random.random() * 2 * math.pi + rotations = random.randint(10, 20) + radius = np.hypot(xx, yy) + angle = np.arctan2(yy, xx) + z = np.zeros_like(radius) + for i in range(rotations): + z += np.cos( + radius * np.sin(angle + i * math.pi * 2 / rotations) * frequency + phase + ) + pixels = np.clip(255 - np.rint(255 * z / rotations), 0, 255).astype(np.uint8) + return Image.fromarray(pixels).convert("RGB") @classmethod def picture(cls, height, width): """ - Create a background with a picture + Create a background with a picture """ - pictures = os.listdir('./pictures') - - if len(pictures) > 0: - picture = Image.open('./pictures/' + pictures[random.randint(0, len(pictures) - 1)]) - - if picture.size[0] < width: - picture = picture.resize([width, int(picture.size[1] * (width / picture.size[0]))], Image.ANTIALIAS) - elif picture.size[1] < height: - picture.thumbnail([int(picture.size[0] * (height / picture.size[1])), height], Image.ANTIALIAS) - - if (picture.size[0] == width): - x = 0 - else: - x = random.randint(0, picture.size[0] - width) - if (picture.size[1] == height): - y = 0 - else: - y = random.randint(0, picture.size[1] - height) - - return picture.crop( - ( - x, - y, - x + width, - y + height, - ) + from PIL import ImageOps + + pictures = [ + os.path.join("./pictures", name) + for name in sorted(os.listdir("./pictures")) + if name.lower().endswith((".jpg", ".jpeg", ".png", ".bmp")) + ] + if not pictures: + raise ValueError("No images found in the pictures folder") + with Image.open(random.choice(pictures)) as picture: + return ImageOps.fit( + picture.convert("RGB"), (width, height), method=Image.Resampling.LANCZOS ) - else: - raise Exception('No images where found in the pictures folder!') diff --git a/imagedt/tools/text_generator/computer_text_generator.py b/imagedt/tools/text_generator/computer_text_generator.py index 4b19bb0..655de1a 100644 --- a/imagedt/tools/text_generator/computer_text_generator.py +++ b/imagedt/tools/text_generator/computer_text_generator.py @@ -1,80 +1,50 @@ +"""Text rasterization using current Pillow font metrics.""" + import random -import imagedt -from PIL import Image, ImageColor, ImageFont, ImageDraw, ImageFilter +from PIL import Image, ImageColor, ImageDraw, ImageFont + -class ComputerTextGenerator(object): +class ComputerTextGenerator: @classmethod def generate(cls, text, font, text_color, font_size, orientation, space_width): + if not text: + raise ValueError("text must be nonempty") + if font_size <= 0 or space_width < 0: + raise ValueError("font size must be positive and space_width nonnegative") + if orientation not in (0, 1): + raise ValueError(f"Unknown orientation {orientation}") + image_font = ImageFont.truetype(font=font, size=int(font_size)) + colors = [ImageColor.getrgb(color) for color in text_color.split(",")] + fill = tuple( + random.randint(min(a, b), max(a, b)) + for a, b in zip(colors[0], colors[-1], strict=True) + ) if orientation == 0: - return cls.__generate_horizontal_text(text, font, text_color, font_size, space_width) - elif orientation == 1: - return cls.__generate_vertical_text(text, font, text_color, font_size, space_width) + words = text.split(" ") + gap = round(image_font.getlength(" ") * space_width) + widths = [max(1, round(image_font.getlength(word))) for word in words] + height = max(1, max(image_font.getbbox(word)[3] for word in words)) + image = Image.new("RGBA", (sum(widths) + gap * (len(words) - 1), height)) + draw = ImageDraw.Draw(image) + x = 0 + for word, width in zip(words, widths, strict=True): + draw.text((x, 0), word, font=image_font, fill=fill) + x += width + gap else: - raise ValueError("Unknown orientation " + str(orientation)) - - @classmethod - def __generate_horizontal_text(cls, text, font, text_color, font_size, space_width): - import random - font_size = random.randint(18, 24) - image_font = ImageFont.truetype(font=font, size=font_size) - words = text.split(' ') - space_width = image_font.getsize(' ')[0] * space_width - - words_width = [image_font.getsize(w)[0] for w in words] - text_width = sum(words_width) + int(space_width) * (len(words) - 1) - text_height = max([image_font.getsize(w)[1] for w in words]) - - txt_img = Image.new('RGBA', (text_width, text_height), (0, 0, 0, 0)) - - txt_draw = ImageDraw.Draw(txt_img) - - colors = [ImageColor.getrgb(c) for c in text_color.split(',')] - c1, c2 = colors[0], colors[-1] - - fill = ( - random.randint(c1[0], c2[0]), - random.randint(c1[1], c2[1]), - random.randint(c1[2], c2[2])) - - sum_words = len(words) - - for i, w in enumerate(words): - if len(w) >= 3: - txt_draw.text((sum(words_width[0:i]) + i * int(space_width), 0), w[:-2], fill=fill, font=image_font) - # last words reset small size - image_font = ImageFont.truetype(font=font, size=font_size/2) - txt_draw.text((sum(words_width[0:i]) + i * int(space_width)+ image_font.getsize(w)[1]*len(w[:-1]) - , 0), w[-2:], fill=fill, font=image_font) - else: - txt_draw.text((sum(words_width[0:i]) + i * int(space_width), 0), w, fill=fill, font=image_font) - return txt_img - - - @classmethod - def __generate_vertical_text(cls, text, font, text_color, font_size, space_width): - image_font = ImageFont.truetype(font=font, size=font_size) - - space_height = int(image_font.getsize(' ')[1] * space_width) - - char_heights = [image_font.getsize(c)[1] if c != ' ' else space_height for c in text] - text_width = max([image_font.getsize(c)[0] for c in text]) - text_height = sum(char_heights) - - txt_img = Image.new('RGBA', (text_width, text_height), (0, 0, 0, 0)) - - txt_draw = ImageDraw.Draw(txt_img) - - colors = [ImageColor.getrgb(c) for c in text_color.split(',')] - c1, c2 = colors[0], colors[-1] - - fill = ( - random.randint(c1[0], c2[0]), - random.randint(c1[1], c2[1]), - random.randint(c1[2], c2[2]) - ) - - for i, c in enumerate(text): - txt_draw.text((0, sum(char_heights[0:i])), c, fill=fill, font=image_font) - - return txt_img + boxes = [image_font.getbbox(char) for char in text] + heights = [ + max(1, box[3]) + if char != " " + else max(1, round(font_size * space_width)) + for char, box in zip(text, boxes, strict=True) + ] + image = Image.new( + "RGBA", (max(1, max(box[2] for box in boxes)), sum(heights)) + ) + draw = ImageDraw.Draw(image) + y = 0 + for char, height in zip(text, heights, strict=True): + draw.text((0, y), char, font=image_font, fill=fill) + y += height + return image diff --git a/imagedt/tools/text_generator/data_generator.py b/imagedt/tools/text_generator/data_generator.py index 03da9ac..800ae76 100644 --- a/imagedt/tools/text_generator/data_generator.py +++ b/imagedt/tools/text_generator/data_generator.py @@ -1,70 +1,95 @@ import os -import sys import random from PIL import Image, ImageFilter -from .computer_text_generator import ComputerTextGenerator -try: - from handwritten_text_generator import HandwrittenTextGenerator -except ImportError as e: - print('Missing modules for handwritten text generation.') - from .background_generator import BackgroundGenerator +from .computer_text_generator import ComputerTextGenerator from .distorsion_generator import DistorsionGenerator -class FakeTextDataGenerator(object): + +class FakeTextDataGenerator: @classmethod def generate_from_tuple(cls, t): """ - Same as generate, but takes all parameters as one tuple + Same as generate, but takes all parameters as one tuple """ return cls.generate(*t) @classmethod - def generate(cls, index, text, font, out_dir, size, extension, skewing_angle, random_skew, - blur, random_blur, background_type, distorsion_type, distorsion_orientation, - is_handwritten, name_format, width, alignment, text_color, orientation, space_width, - save=False): + def generate( + cls, + index, + text, + font, + out_dir, + size, + extension, + skewing_angle, + random_skew, + blur, + random_blur, + background_type, + distorsion_type, + distorsion_orientation, + is_handwritten, + name_format, + width, + alignment, + text_color, + orientation, + space_width, + save=False, + ): image = None ########################## # Create picture of text # ########################## - # remove handwirtten style + # remove handwirtten style # if is_handwritten: # if orientation == 1: # raise ValueError("Vertical handwritten text is unavailable") # image = HandwrittenTextGenerator.generate(text) # else: - image = ComputerTextGenerator.generate(text, font, text_color, size, orientation, space_width) + if size <= 10 or width < 0: + raise ValueError("size must exceed 10 and width must be nonnegative") + if is_handwritten: + raise NotImplementedError( + "Handwritten generation requires external pretrained assets and is not supported by this generator" + ) + image = ComputerTextGenerator.generate( + text, font, text_color, size, orientation, space_width + ) - random_angle = random.randint(0-skewing_angle, skewing_angle) + random_angle = random.randint(0 - skewing_angle, skewing_angle) - rotated_img = image.rotate(skewing_angle if not random_skew else random_angle, expand=1) + rotated_img = image.rotate( + skewing_angle if not random_skew else random_angle, expand=1 + ) ############################# # Apply distorsion to image # ############################# if distorsion_type == 0: - distorted_img = rotated_img # Mind = blown + distorted_img = rotated_img # Mind = blown elif distorsion_type == 1: distorted_img = DistorsionGenerator.sin( rotated_img, vertical=(distorsion_orientation == 0 or distorsion_orientation == 2), - horizontal=(distorsion_orientation == 1 or distorsion_orientation == 2) + horizontal=(distorsion_orientation == 1 or distorsion_orientation == 2), ) elif distorsion_type == 2: distorted_img = DistorsionGenerator.cos( rotated_img, vertical=(distorsion_orientation == 0 or distorsion_orientation == 2), - horizontal=(distorsion_orientation == 1 or distorsion_orientation == 2) + horizontal=(distorsion_orientation == 1 or distorsion_orientation == 2), ) else: distorted_img = DistorsionGenerator.random( rotated_img, vertical=(distorsion_orientation == 0 or distorsion_orientation == 2), - horizontal=(distorsion_orientation == 1 or distorsion_orientation == 2) + horizontal=(distorsion_orientation == 1 or distorsion_orientation == 2), ) ################################## @@ -73,14 +98,24 @@ def generate(cls, index, text, font, out_dir, size, extension, skewing_angle, ra # Horizontal text if orientation == 0: - new_width = int(float(distorted_img.size[0] + 10) * (float(size) / float(distorted_img.size[1] + 10))) - resized_img = distorted_img.resize((new_width, size - 10), Image.ANTIALIAS) + new_width = int( + float(distorted_img.size[0] + 10) + * (float(size) / float(distorted_img.size[1] + 10)) + ) + resized_img = distorted_img.resize( + (new_width, size - 10), Image.Resampling.LANCZOS + ) background_width = width if width > 0 else new_width + 10 background_height = size # Vertical text elif orientation == 1: - new_height = int(float(distorted_img.size[1] + 10) * (float(size) / float(distorted_img.size[0] + 10))) - resized_img = distorted_img.resize((size - 10, new_height), Image.ANTIALIAS) + new_height = int( + float(distorted_img.size[1] + 10) + * (float(size) / float(distorted_img.size[0] + 10)) + ) + resized_img = distorted_img.resize( + (size - 10, new_height), Image.Resampling.LANCZOS + ) background_width = size background_height = new_height + 10 else: @@ -89,16 +124,22 @@ def generate(cls, index, text, font, out_dir, size, extension, skewing_angle, ra ############################# # Generate background image # ############################# - background_height = 96 - background_width = 200 if background_type == 0: - background = BackgroundGenerator.gaussian_noise(background_height, background_width) + background = BackgroundGenerator.gaussian_noise( + background_height, background_width + ) elif background_type == 1: - background = BackgroundGenerator.plain_white(background_height, background_width) + background = BackgroundGenerator.plain_white( + background_height, background_width + ) elif background_type == 2: - background = BackgroundGenerator.quasicrystal(background_height, background_width) + background = BackgroundGenerator.quasicrystal( + background_height, background_width + ) else: - background = BackgroundGenerator.picture(background_height, background_width) + background = BackgroundGenerator.picture( + background_height, background_width + ) ############################# # Place text with alignment # @@ -109,9 +150,15 @@ def generate(cls, index, text, font, out_dir, size, extension, skewing_angle, ra if alignment == 0: background.paste(resized_img, (5, 5), resized_img) elif alignment == 1: - background.paste(resized_img, (int(background_width / 2 - new_text_width / 2), 5), resized_img) + background.paste( + resized_img, + (int(background_width / 2 - new_text_width / 2), 5), + resized_img, + ) else: - background.paste(resized_img, (background_width - new_text_width - 5, 5), resized_img) + background.paste( + resized_img, (background_width - new_text_width - 5, 5), resized_img + ) ################################## # Apply gaussian blur # @@ -127,19 +174,17 @@ def generate(cls, index, text, font, out_dir, size, extension, skewing_angle, ra # Generate name for resulting image # ##################################### if name_format == 0: - # python 2: need encode unicode - if sys.hexversion < 0x3000000: - text = text.encode('utf-8') - image_name = '{}_{}.{}'.format(text, str(index), extension) + image_name = f"{text}_{str(index)}.{extension}" elif name_format == 1: - image_name = '{}_{}.{}'.format(str(index), text, extension) + image_name = f"{str(index)}_{text}.{extension}" elif name_format == 2: - image_name = '{}.{}'.format(str(index).zfill(8),extension) + image_name = f"{str(index).zfill(8)}.{extension}" else: - print('{} is not a valid name format. Using default.'.format(name_format)) - image_name = '{}_{}.{}'.format(text, str(index), extension) + print(f"{name_format} is not a valid name format. Using default.") + image_name = f"{text}_{str(index)}.{extension}" # Save the image if save: - final_image.convert('RGB').save(os.path.join(out_dir, image_name)) + os.makedirs(out_dir, exist_ok=True) + final_image.convert("RGB").save(os.path.join(out_dir, image_name)) return final_image, text diff --git a/imagedt/tools/text_generator/distorsion_generator.py b/imagedt/tools/text_generator/distorsion_generator.py index 67072f1..c1336b7 100644 --- a/imagedt/tools/text_generator/distorsion_generator.py +++ b/imagedt/tools/text_generator/distorsion_generator.py @@ -1,85 +1,108 @@ -import cv2 import math -import os import random + import numpy as np +from PIL import Image -from PIL import Image, ImageDraw, ImageFilter -class DistorsionGenerator(object): +class DistorsionGenerator: @classmethod def apply_func_distorsion(cls, image, vertical, horizontal, max_offset, func): - """ - """ + """ """ # Nothing to do! if not vertical and not horizontal: return image - rgb_image = image.convert('RGBA') - + rgb_image = image.convert("RGBA") + img_arr = np.array(rgb_image) - vertical_offsets = [func(i) for i in range(img_arr.shape[1])] + vertical_offsets = [int(func(i)) for i in range(img_arr.shape[1])] horizontal_offsets = [ func(i) - for i in range( - img_arr.shape[0] + ( - (max(vertical_offsets) - min(min(vertical_offsets), 0)) if vertical else 0 - ) - ) + for i in range(img_arr.shape[0] + (2 * max_offset if vertical else 0)) ] - new_img_arr = np.zeros(( - img_arr.shape[0] + (2 * max_offset if vertical else 0), - img_arr.shape[1] + (2 * max_offset if horizontal else 0), - 4 - )) + new_img_arr = np.zeros( + ( + img_arr.shape[0] + (2 * max_offset if vertical else 0), + img_arr.shape[1] + (2 * max_offset if horizontal else 0), + 4, + ), + dtype=np.uint8, + ) new_img_arr_copy = np.copy(new_img_arr) - + if vertical: column_height = img_arr.shape[0] for i, o in enumerate(vertical_offsets): column_pos = (i + max_offset) if horizontal else i - new_img_arr[max_offset+o:column_height+max_offset+o, column_pos, :] = img_arr[:, i, :] + new_img_arr[ + max_offset + o : column_height + max_offset + o, column_pos, : + ] = img_arr[:, i, :] if horizontal: row_width = img_arr.shape[1] for i, o in enumerate(horizontal_offsets): if vertical: - new_img_arr_copy[i, max_offset+o:row_width+max_offset+o,:] = new_img_arr[i, max_offset:row_width+max_offset, :] + new_img_arr_copy[ + i, max_offset + o : row_width + max_offset + o, : + ] = new_img_arr[i, max_offset : row_width + max_offset, :] else: - new_img_arr[i, max_offset+o:row_width+max_offset+o,:] = img_arr[i, :, :] + new_img_arr[i, max_offset + o : row_width + max_offset + o, :] = ( + img_arr[i, :, :] + ) - return Image.fromarray(np.uint8(new_img_arr_copy if horizontal and vertical else new_img_arr)).convert('RGBA') + return Image.fromarray( + np.uint8(new_img_arr_copy if horizontal and vertical else new_img_arr) + ).convert("RGBA") @classmethod def sin(cls, image, vertical=False, horizontal=False): """ - Apply a sine distorsion on one or both of the specified axis + Apply a sine distorsion on one or both of the specified axis """ - max_offset = int(image.height ** 0.5) + max_offset = int(image.height**0.5) - return cls.apply_func_distorsion(image, vertical, horizontal, max_offset, (lambda x: int(math.sin(math.radians(x)) * max_offset))) + return cls.apply_func_distorsion( + image, + vertical, + horizontal, + max_offset, + (lambda x: int(math.sin(math.radians(x)) * max_offset)), + ) @classmethod def cos(cls, image, vertical=False, horizontal=False): """ - Apply a cosine distorsion on one or both of the specified axis + Apply a cosine distorsion on one or both of the specified axis """ - max_offset = int(image.height ** 0.5) + max_offset = int(image.height**0.5) - return cls.apply_func_distorsion(image, vertical, horizontal, max_offset, (lambda x: int(math.cos(math.radians(x)) * max_offset))) + return cls.apply_func_distorsion( + image, + vertical, + horizontal, + max_offset, + (lambda x: int(math.cos(math.radians(x)) * max_offset)), + ) @classmethod def random(cls, image, vertical=False, horizontal=False): """ - Apply a random distorsion on one or both of the specified axis + Apply a random distorsion on one or both of the specified axis """ - max_offset = int(image.height ** 0.4) + max_offset = int(image.height**0.4) - return cls.apply_func_distorsion(image, vertical, horizontal, max_offset, (lambda x: random.randint(0, max_offset))) + return cls.apply_func_distorsion( + image, + vertical, + horizontal, + max_offset, + (lambda x: random.randint(0, max_offset)), + ) diff --git a/imagedt/tools/text_generator/handwritten_text_generator.py b/imagedt/tools/text_generator/handwritten_text_generator.py index 11c5f43..af950c8 100644 --- a/imagedt/tools/text_generator/handwritten_text_generator.py +++ b/imagedt/tools/text_generator/handwritten_text_generator.py @@ -1,153 +1,10 @@ -import os -import pickle -import numpy as np -import tensorflow as tf -import matplotlib.pyplot as plt -import matplotlib.cm as cm -import matplotlib.mlab as mlab -import seaborn -from PIL import Image -from collections import namedtuple +"""Compatibility entry for an unsupported pretrained handwriting pipeline.""" -class HandwrittenTextGenerator(object): - @classmethod - def __sample(cls, e, mu1, mu2, std1, std2, rho): - cov = np.array([[std1 * std1, std1 * std2 * rho], - [std1 * std2 * rho, std2 * std2]]) - mean = np.array([mu1, mu2]) - - x, y = np.random.multivariate_normal(mean, cov) - end = np.random.binomial(1, e) - return np.array([x, y, end]) - - - @classmethod - def __split_strokes(cls, points): - points = np.array(points) - strokes = [] - b = 0 - for e in range(len(points)): - if points[e, 2] == 1.: - strokes += [points[b: e + 1, :2].copy()] - b = e + 1 - return strokes - - - @classmethod - def __cumsum(cls, points): - sums = np.cumsum(points[:, :2], axis=0) - return np.concatenate([sums, points[:, 2:]], axis=1) - - - @classmethod - def __sample_text(cls, sess, args_text, translation): - # Original creator said it helps (https://github.com/Grzego/handwriting-generation/issues/3) - args_text += ' ' - - fields = ['coordinates', 'sequence', 'bias', 'e', 'pi', 'mu1', 'mu2', 'std1', 'std2', - 'rho', 'window', 'kappa', 'phi', 'finish', 'zero_states'] - vs = namedtuple('Params', fields)( - *[tf.get_collection(name)[0] for name in fields] - ) - - text = np.array([translation.get(c, 0) for c in args_text]) - sequence = np.eye(len(translation), dtype=np.float32)[text] - sequence = np.expand_dims(np.concatenate([sequence, np.zeros((1, len(translation)))]), axis=0) - - coord = np.array([0., 0., 1.]) - coords = [coord] - - phi_data, window_data, kappa_data, stroke_data = [], [], [], [] - sess.run(vs.zero_states) - for s in range(1, 60 * len(args_text) + 1): - e, pi, mu1, mu2, std1, std2, rho, \ - finish, phi, window, kappa = sess.run([vs.e, vs.pi, vs.mu1, vs.mu2, - vs.std1, vs.std2, vs.rho, vs.finish, - vs.phi, vs.window, vs.kappa], - feed_dict={ - vs.coordinates: coord[None, None, ...], - vs.sequence: sequence, - vs.bias: 1. - }) - phi_data += [phi[0, :]] - window_data += [window[0, :]] - kappa_data += [kappa[0, :]] - # --- - g = np.random.choice(np.arange(pi.shape[1]), p=pi[0]) - coord = cls.__sample(e[0, 0], mu1[0, g], mu2[0, g], - std1[0, g], std2[0, g], rho[0, g]) - coords += [coord] - stroke_data += [[mu1[0, g], mu2[0, g], std1[0, g], std2[0, g], rho[0, g], coord[2]]] - - if finish[0, 0] > 0.8: - break - - coords = np.array(coords) - coords[-1, 2] = 1. - - return phi_data, window_data, kappa_data, stroke_data, coords - - @classmethod - def __crop_white_borders(cls, image): - image_data = np.asarray(image) - non_empty_columns = np.where(image_data.min(axis=0)<255)[0] - non_empty_rows = np.where(image_data.min(axis=1)<255)[0] - cropBox = (min(non_empty_rows), max(non_empty_rows), min(non_empty_columns), max(non_empty_columns)) - - image_data_new = image_data[cropBox[0]:cropBox[1]+1, cropBox[2]:cropBox[3]+1] - - return Image.fromarray(image_data_new) - - @classmethod - def __join_images(cls, images): - widths, heights = zip(*(i.size for i in images)) - - total_width = sum(widths) - 35 * len(images) - max_height = max(heights) - - compound_image = Image.new('L', (total_width, max_height)) - - x_offset = 0 - for im in images: - compound_image.paste(im, (x_offset,0)) - x_offset += (im.size[0] - 35) - - return compound_image +class HandwrittenTextGenerator: @classmethod def generate(cls, text): - with open(os.path.join('handwritten_model', 'translation.pkl'), 'rb') as file: - translation = pickle.load(file) - - config = tf.ConfigProto( - device_count={'GPU': 0} + raise NotImplementedError( + "The legacy handwriting pipeline depends on an unbundled TensorFlow 1 model. " + "Use ComputerTextGenerator or supply a maintained handwriting backend." ) - tf.reset_default_graph() - with tf.Session(config=config) as sess: - saver = tf.train.import_meta_graph('handwritten_model/model-29.meta') - saver.restore(sess, 'handwritten_model/model-29') - images = [] - for word in text.split(' '): - _, window_data, kappa_data, stroke_data, coords = cls.__sample_text(sess, word, translation) - - strokes = np.array(stroke_data) - strokes[:, :2] = np.cumsum(strokes[:, :2], axis=0) - _, maxx = np.min(strokes[:, 0]), np.max(strokes[:, 0]) - miny, maxy = np.min(strokes[:, 1]), np.max(strokes[:, 1]) - - fig, ax = plt.subplots(1, 1) - fig.patch.set_visible(False) - ax.axis('off') - - for stroke in cls.__split_strokes(cls.__cumsum(np.array(coords))): - plt.plot(stroke[:, 0], -stroke[:, 1], color='#080808') - - canvas = plt.get_current_fig_manager().canvas - canvas.draw() - - image = Image.frombytes('RGB', canvas.get_width_height(), canvas.tostring_rgb()).convert('L') - images.append(cls.__crop_white_borders(image)) - - plt.close() - - return cls.__join_images(images) diff --git a/imagedt/tools/text_generator/text_generator.py b/imagedt/tools/text_generator/text_generator.py index 2b98b14..e2d6695 100644 --- a/imagedt/tools/text_generator/text_generator.py +++ b/imagedt/tools/text_generator/text_generator.py @@ -1,129 +1,137 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - import os -import sys -import time import random -import argparse -import numpy as np -from tqdm import tqdm +from multiprocessing.dummy import Pool -if sys.hexversion < 0x3000000: - from multiprocessing.dummy import Pool -else: - from multiprocessing import Pool +import numpy as np from .data_generator import FakeTextDataGenerator -class Text_Generator(object): - """docstring for Text_Generator""" - def __init__(self, font_path, langu='cn'): - super(Text_Generator, self).__init__() - self.langu = langu - self.font_path = font_path - self.fonts = self._load_fonts - self._init_multi_threads - self.set_params() - - @property - def _load_fonts(self): - """ - Load all fonts in the fonts directories - """ - # if self.langu == 'cn': - # cn_font_dir = os.path.join(self.font_dir, 'Chinese_font') - # return [os.path.join(cn_font_dir, font) for font in os.listdir(cn_font_dir)] - # else: - # en_font_dir = os.path.join(self.font_dir, 'English_font') - # return [os.path.join(en_font_dir, font) for font in os.listdir(en_font_dir)] - return [os.path.join(self.font_path, font) for font in os.listdir(self.font_path)] - - def set_params(self, tests=None): - if tests is not None: - count = len(tests) - text = tests - else: - count = 64 - text = [u'测试快得飞起'] *count - - index = [i for i in range(0, count)] - font_path = self.fonts[:1] * count - save_dir = ['/data/tmp/temp/price_tags_fake_dats'] * count - size = [96] * count - extension = ['png'] * count - skewing_angle = [random.randint(0, 20) for _ in range(count)] - # skewing_angle = [0] * count - random_skew = [True] * count - blur = [0] * count - random_blur = [True] * count - background_type = [0] * count - distorsion_type = [0] * count - distorsion_orientation = [0] * count - is_handwritten = [False] * count - name_format = [1] * count - width = [200] * count - alignment = [1] * count - text_color = ['#282828'] * count - orientation = [0] * count - space_width = [1.0] * count - save = [False] * count - - self.args = zip(index, - text, - font_path, - save_dir, - size, - extension, - skewing_angle, - random_skew, - blur, - random_blur, - background_type, - distorsion_type, - distorsion_orientation, - is_handwritten, - name_format, - width, - alignment, - text_color, - orientation, - space_width, - save) - - @property - def _init_multi_threads(self, num_theards=4): - self.pool = Pool(num_theards) - - def generate_texts(self): - show_progress = False - st_time = time.time() - if show_progress: - [items for items in tqdm(self.pool.imap_unordered(FakeTextDataGenerator.generate_from_tuple, self.args), - total=count)] - else: - [items for items in self.pool.imap_unordered(FakeTextDataGenerator.generate_from_tuple, self.args)] - self.pool.terminate() - print ("time cost: {0:.4f}".format(time.time()-st_time)) - - # import imagedt - # @imagedt.decorator.time_cost - def gen_price_realtime_datas(self, batch_size=64): - items, texts = [], [] - for _ in xrange(batch_size): - text = str(random.randint(1, 99999)) - if len(text) > 2: - text = text[:-1] + '0' - texts.append(text) - - self.set_params(texts) - # for _ in tqdm(self.pool.imap_unordered(FakeTextDataGenerator.generate_from_tuple, self.args), - # total=len(texts)): - # pass - items = [[np.array(items[0]), items[1]] for items in - self.pool.imap_unordered(FakeTextDataGenerator.generate_from_tuple, self.args)] - # close - # self.pool.terminate() - return items +class Text_Generator: + """docstring for Text_Generator""" + + def __init__(self, font_path, langu="cn"): + super().__init__() + self.langu = langu + self.font_path = font_path + self.fonts = self._load_fonts + self.pool = self._init_multi_threads + self.set_params() + + @property + def _load_fonts(self): + """ + Load all fonts in the fonts directories + """ + # if self.langu == 'cn': + # cn_font_dir = os.path.join(self.font_dir, 'Chinese_font') + # return [os.path.join(cn_font_dir, font) for font in os.listdir(cn_font_dir)] + # else: + # en_font_dir = os.path.join(self.font_dir, 'English_font') + # return [os.path.join(en_font_dir, font) for font in os.listdir(en_font_dir)] + fonts = [ + os.path.join(self.font_path, font) + for font in sorted(os.listdir(self.font_path)) + if font.lower().endswith((".ttf", ".otf", ".ttc")) + ] + if not fonts: + raise ValueError("No font files found") + return fonts + + def set_params(self, tests=None): + if tests is not None: + count = len(tests) + text = tests + else: + count = 64 + text = ["测试快得飞起"] * count + + index = [i for i in range(0, count)] + font_path = self.fonts[:1] * count + save_dir = ["/data/tmp/temp/price_tags_fake_dats"] * count + size = [96] * count + extension = ["png"] * count + skewing_angle = [random.randint(0, 20) for _ in range(count)] + # skewing_angle = [0] * count + random_skew = [True] * count + blur = [0] * count + random_blur = [True] * count + background_type = [0] * count + distorsion_type = [0] * count + distorsion_orientation = [0] * count + is_handwritten = [False] * count + name_format = [1] * count + width = [200] * count + alignment = [1] * count + text_color = ["#282828"] * count + orientation = [0] * count + space_width = [1.0] * count + save = [False] * count + + self.args = list( + zip( + index, + text, + font_path, + save_dir, + size, + extension, + skewing_angle, + random_skew, + blur, + random_blur, + background_type, + distorsion_type, + distorsion_orientation, + is_handwritten, + name_format, + width, + alignment, + text_color, + orientation, + space_width, + save, + strict=True, + ) + ) + + @property + def _init_multi_threads(self, num_theards=4): + return Pool(num_theards) + + def generate_texts(self): + return self.pool.map(FakeTextDataGenerator.generate_from_tuple, self.args) + + def close(self): + self.pool.close() + self.pool.join() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def gen_price_realtime_datas(self, batch_size=64): + if not isinstance(batch_size, int) or batch_size < 0: + raise ValueError("batch_size must be a nonnegative integer") + items, texts = [], [] + for _ in range(batch_size): + text = str(random.randint(1, 99999)) + if len(text) > 2: + text = text[:-1] + "0" + texts.append(text) + + self.set_params(texts) + # for _ in tqdm(self.pool.imap(FakeTextDataGenerator.generate_from_tuple, self.args), + # total=len(texts)): + # pass + items = [ + [np.array(items[0]), items[1]] + for items in self.pool.imap( + FakeTextDataGenerator.generate_from_tuple, self.args + ) + ] + # close + # self.pool.terminate() + return items diff --git a/imagedt/torch/__init__.py b/imagedt/torch/__init__.py index f8625f5..efd41aa 100644 --- a/imagedt/torch/__init__.py +++ b/imagedt/torch/__init__.py @@ -1,7 +1,3 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function - -from . import network -from . import optim -from .trainer import * \ No newline at end of file +from . import network as network +from . import optim as optim +from . import trainer as trainer diff --git a/imagedt/torch/network/__init__.py b/imagedt/torch/network/__init__.py index 760c836..e69de29 100644 --- a/imagedt/torch/network/__init__.py +++ b/imagedt/torch/network/__init__.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function diff --git a/imagedt/torch/optim/__init__.py b/imagedt/torch/optim/__init__.py index 760c836..e69de29 100644 --- a/imagedt/torch/optim/__init__.py +++ b/imagedt/torch/optim/__init__.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function diff --git a/imagedt/torch/trainer.py b/imagedt/torch/trainer.py index 913ef8f..e69de29 100644 --- a/imagedt/torch/trainer.py +++ b/imagedt/torch/trainer.py @@ -1,3 +0,0 @@ -# coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2af6151 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "imagedt" +version = "0.6.0rc1" +description = "Utilities for image processing, annotation, evaluation and model conversion" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [{name = "pytorch_fans11", email = "zhengyzms@163.com"}] +dependencies = ["numpy>=1.23", "Pillow>=10", "opencv-python-headless>=4.8", "lxml>=4.9"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/hanranCode/ImageDT" + +[project.optional-dependencies] +plot = ["matplotlib>=3.7"] +excel = ["xlwt>=1.3"] +pdf = ["pypdfium2>=4.30"] +tensorflow = ["tensorflow>=2.16,<3", "protobuf>=4.25", "ai-edge-litert>=1.2"] +lite = ["ai-edge-litert>=1.2"] +caffe = ["protobuf>=4.25"] +dev = ["pytest>=8", "pytest-cov>=5", "build>=1", "twine>=5", "ruff>=0.17,<0.18", "setuptools>=77"] + +[tool.setuptools.packages.find] +include = ["imagedt*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" +markers = ["tensorflow: requires the TensorFlow extra", "integration: executes a real optional backend"] + +[tool.coverage.run] +branch = true +source = ["imagedt"] +omit = ["*/caffe_pb2.py", "*/test_lenet.py"] + +[tool.ruff] +target-version = "py310" +exclude = ["imagedt/converters/taffe/caffe/caffe_pb2.py"] +line-length = 88 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I", "B", "UP"] + +[tool.ruff.lint.per-file-ignores] +# Optional backend must be skipped before importing backend-specific modules. +"tests/test_tensorflow.py" = ["E402"] diff --git a/requirements.txt b/requirements.txt index 5aad2fc..fd7f606 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1 @@ -numpy>=1.8.1,<=1.11.0 -lxml==4.2.5 -protobuf==2.6.1 - \ No newline at end of file +-e .[plot,excel,pdf] diff --git a/scripts/benchmark_utils.py b/scripts/benchmark_utils.py new file mode 100644 index 0000000..656f5f9 --- /dev/null +++ b/scripts/benchmark_utils.py @@ -0,0 +1,52 @@ +"""Reproduce the ROC threshold scan comparison against the legacy algorithm.""" + +import json +from time import perf_counter + +import numpy as np + + +def benchmark(samples=5000): + rng = np.random.default_rng(42) + probabilities = rng.random(samples) + matches = rng.integers(0, 2, samples) + thresholds = np.arange(0, 1.001, 0.001) + start = perf_counter() + expected = [] + for threshold in thresholds: + tp = fp = 0 + for probability, match in zip(probabilities, matches, strict=True): + if probability >= threshold: + if match: + tp += 1 + else: + fp += 1 + expected.append( + (tp / max(int(matches.sum()), 1), fp / max(int((matches == 0).sum()), 1)) + ) + legacy_seconds = perf_counter() - start + start = perf_counter() + positive = np.sort(probabilities[matches == 1]) + negative = np.sort(probabilities[matches == 0]) + actual = np.column_stack( + ( + (len(positive) - np.searchsorted(positive, thresholds)) + / max(len(positive), 1), + (len(negative) - np.searchsorted(negative, thresholds)) + / max(len(negative), 1), + ) + ) + new_seconds = perf_counter() - start + np.testing.assert_allclose(actual, expected) + return { + "samples": samples, + "thresholds": len(thresholds), + "legacy_seconds": legacy_seconds, + "new_seconds": new_seconds, + "speedup": legacy_seconds / new_seconds, + "numerical_equivalence": True, + } + + +if __name__ == "__main__": + print(json.dumps(benchmark(), indent=2)) diff --git a/scripts/check_release.py b/scripts/check_release.py new file mode 100644 index 0000000..3f0ec9f --- /dev/null +++ b/scripts/check_release.py @@ -0,0 +1,62 @@ +"""Check release metadata and source distribution completeness without uploading.""" + +import hashlib +import json +import tarfile +import zipfile +from email.parser import BytesParser +from pathlib import Path + + +def check(directory): + wheels = list(Path(directory).glob("imagedt-*.whl")) + sources = list(Path(directory).glob("imagedt-*.tar.gz")) + if len(wheels) != 1 or len(sources) != 1: + raise ValueError("Expected exactly one wheel and one source distribution") + with zipfile.ZipFile(wheels[0]) as wheel: + names = wheel.namelist() + metadata_path = next( + name for name in names if name.endswith(".dist-info/METADATA") + ) + metadata = BytesParser().parsebytes(wheel.read(metadata_path)) + assert metadata["Name"] == "imagedt" + assert metadata["Requires-Python"] == ">=3.10" + assert ( + "Tag: py3-none-any" + in wheel.read( + next(name for name in names if name.endswith(".dist-info/WHEEL")) + ).decode() + ) + assert not any(name.endswith(".pyc") for name in names) + for name in names: + if name.endswith(".py"): + compile(wheel.read(name), name, "exec") + with tarfile.open(sources[0]) as source: + names = source.getnames() + for required in [ + "tests/fixtures/legacy_api.json", + "pyproject.toml", + "docs/migration.md", + "README.md", + ]: + assert any(name.endswith("/" + required) for name in names), ( + f"Missing {required}" + ) + return { + "version": metadata["Version"], + "artifacts": [ + { + "path": str(path), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "bytes": path.stat().st_size, + } + for path in [wheels[0], sources[0]] + ], + "metadata_and_source_checks": "passed", + } + + +if __name__ == "__main__": + import sys + + print(json.dumps(check(sys.argv[1] if len(sys.argv) > 1 else "dist"), indent=2)) diff --git a/setup.py b/setup.py index b0ce987..7ff7d9b 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,5 @@ -#!/usr/bin/env python -from setuptools import setup, find_packages +"""Compatibility entry point; package metadata lives in pyproject.toml.""" -setup( - name = 'imagedt', - version = '0.4.1', - keywords = ('imagedt'), - description = 'a python lib for neural networks, file and image processing etc. ', - long_description="##[Details](https://github.com/hanranCode/ImageDT)", - long_description_content_type="text/markdown", - license = 'Apache License 2.0', +from setuptools import setup - url = 'https://github.com/hanranCode/ImageDT', - author = 'pytorch_fans11', - author_email = 'zhengyzms@163.com', - - packages = find_packages(), - include_package_data = True, - platforms = 'any', - install_requires = [], -) +setup() diff --git a/test/Test_converte_model.py b/test/Test_converte_model.py index 76485b8..a26a544 100644 --- a/test/Test_converte_model.py +++ b/test/Test_converte_model.py @@ -1,55 +1,19 @@ -#!/usr/bin/env python -# coding: utf-8 -import gc -import os -os.environ['CUDA_VISIBLE_DEVICES'] = '1' -# from nose.tools import * -import sys -sys.path.append('./') - -from imagedt.converters import caffe_tfmodel -from imagedt.converters import caffe_coreml -from imagedt.converters import tfmodel_lite -from imagedt.decorator import time_cost - -class Test_Model_Converte(object): - def __init__(self): - super(Test_Model_Converte, self).__init__() - - @time_cost - def caffe_tfmodel_convert(self, proto_def_path, caffemodel_path, output_model_name='freeze_graph_model.pb'): - caffe_tfmodel.convert(proto_def_path, caffemodel_path) - - def test_caffe_tfmodel_convert(self): - proto_def_path = '/data/dataset/liby_offline/train_renet50/models/deploy.prototxt' - caffemodel_path = '/data/dataset/liby_offline/train_renet50/models/resnet50_snapshot_iter_60000.caffemodel' - self.caffe_tfmodel_convert(proto_def_path, caffemodel_path) - - @time_cost - def caffe_coreml_convert(self, model_path, prototxt_path, label_path): - caffe_coreml.convert(model_path, prototxt_path, label_path, red=-123, green=-117, blue=-104, - scale=1.0, bgr=False, output_model_name='sku_cls_model_noise.mlmodel') +"""Legacy test entry point; runs the maintained Python 3 regression suite.""" - def test_caffe_coreml_convert(self): - prototxt_path = '/data/dataset/liby_offline/train_renet50/models/deploy.prototxt' - model_path = '/data/dataset/liby_offline/train_renet50/models/resnet50_snapshot_iter_60000.caffemodel' - label_path = '/data/dataset/liby_offline/train_renet50/models/sku_labels.txt' - self.caffe_coreml_convert(model_path, prototxt_path, label_path) - - def tfmodel_lite_convert(self, graph_def_file, output_file): - tfmodel_lite.convert(graph_def_file, output_file, output_format='TFLITE', inference_type='FLOAT', inference_input_type='FLOAT', - input_array='data', input_shape='1,224,224,3', output_arrays='softmax') - - @time_cost - def test_tfmodel_lite_convert(self): - # TODO fix bug: pipeline excu program - graph_def_file = '/data/project/idt-jackfruit-mobile-detect/data/ssd_mobile_512_liby_1/shel_detect_model.ckpt.pb' - output_file = '/data/project/idt-jackfruit-mobile-detect/data/ssd_mobile_512_liby_1/shel_detect_model.tflite' - self.tfmodel_lite_convert(graph_def_file, output_file) - - -if __name__ == '__main__': - Converte_init = Test_Model_Converte() - Converte_init.test_caffe_coreml_convert() - # Converte_init.test_caffe_tfmodel_convert() - # Converte_init.test_tfmodel_lite_convert() +import subprocess +import sys +from pathlib import Path + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + raise SystemExit( + subprocess.call( + [ + sys.executable, + "-m", + "pytest", + str(root / "tests" / "test_caffe_backends.py"), + ], + cwd=root, + ) + ) diff --git a/test/Test_file_operators.py b/test/Test_file_operators.py index 03f3872..fd3e1de 100644 --- a/test/Test_file_operators.py +++ b/test/Test_file_operators.py @@ -1,105 +1,14 @@ -#!/usr/bin/env python -# coding: utf-8 +"""Legacy test entry point; runs the maintained Python 3 regression suite.""" -import os +import subprocess import sys -import nose.tools as ntools - -from imagedt.file import parse_annotation, file_operate -from imagedt.tensorflow.tools import data_converters -from imagedt.dir import dir_loop -from imagedt.image import process -from imagedt.tools import detect_eval -from imagedt.file import file_write - - -class Test_File_operators(object): - def __init__(self): - super(Test_File_operators, self).__init__() - self.parse_tools = parse_annotation.Anno_OP - self.file_op = file_operate.FilesOp - self.test_data_dir = './test/sources/data_dir' - self.test_train_datas_dir = './test/sources/train_datas_dir' - - def _create_brk_jpg(self): - create_brk_jpg = os.path.join(self.test_data_dir, 'broken.jpg') - with open(create_brk_jpg, 'w') as f: - f.write("'It's boring") - return create_brk_jpg - - def parse_annotations(self, xml_path): - self.parse_tools.parse_lab_xml(xml_path) - - def test_pars_annotations(self): - xml_poaths = dir_loop.loop(self.test_data_dir, ['.xml']) - self.parse_annotations(xml_poaths[0]) - - def test_converte_detect_records(self): - data_dir = '/ssd_data/price_tag/detect' - save_path = os.path.join(data_dir, './training.tfrecord') - data_converters.converte_detect_records(data_dir, save_path) - - ###################### test_file_operators ###################### - def test_rename_file_with_uuid(self): - self.file_op.rename_file_with_uuid(self.test_data_dir) - - def test_del_broken_image(self): - # test case: creating a new broken image file , it should be delete - create_brk_jpg = self._create_brk_jpg() - ntools.assert_true(os.path.isfile(create_brk_jpg)) - # check broken images - self.file_op.del_broken_image(self.test_data_dir) - # deleted file - ntools.assert_true(os.path.isfile(create_brk_jpg)) - - def test_check_data_pairs(self): - self.file_op.check_data_pairs(self.test_data_dir) - - def test_file_operate_one(self): - self.file_op.rename_class_dir(self.test_train_datas_dir, before_cls_str='class1') - classes = os.listdir(self.test_train_datas_dir)[0] - ntools.assert_false(classes.startswith('other_')) - self.file_op.rename_class_dir(self.test_train_datas_dir, before_cls_str='other_diao_') - classes = os.listdir(self.test_train_datas_dir)[0] - ntools.assert_true(classes.startswith('other')) - ###################### test_file_operators ###################### - - def test_process_remove_brokken(self): - create_brk_jpg = self._create_brk_jpg() - ntools.assert_true(os.path.isfile(create_brk_jpg)) - process.remove_broken_image(self.test_data_dir) - # deleted file - ntools.assert_false(os.path.isfile(create_brk_jpg)) - - def rename_xml_cls_name(self): - self.parse_tools.reset_xml_cls('/data/dataset/shelf_datasets/test_detect_save/Annotations', name='3477', desc=u'圆柱圆台形') - - def test_write_xml_with_boxandscore(self): - # bndboxs : [[xmin, ymin, xmax, ymax], [xmin, ymin, xmax, ymax], ...] - # scores: [0.8, 0.9, score3, score4, score5] - # len(bndboxs) == len(scores) - # xmlname: save xml path,eg:"save_dir/xxxxx.xml" - bndboxs = [[1,1, 101, 101], [11,11, 1101, 1101]] - scores = [0.9, 0.1] - xmlname = os.path.join(os.path.dirname(self.test_train_datas_dir), 'text_write.xml') - self.parse_tools.write_xml(bndboxs, scores, xmlname, thresh=0.2, classes='3488') - - def metric_f1_score(self): - # detpath = '/data/tmp/det_txt' - detpath = '/data/dataset/pg_one/PG/PG_data0726v2/models/train_datasets_jutter_10/PG_Densenet_161_origin /train_datasets_jutter_10/Annotations' - annopath = '/data/dataset/pg_one/PG/PG_data0726v2/test/Annotations' - f1_score = detect_eval.voc_eval(detpath, annopath, ovthresh=0.5, det_file_type='xml', conf=0.5) - - print 'mean f1-score: {0}'.format(f1_score) - - -if __name__ == '__main__': - File_operate_init = Test_File_operators() - # File_operate_init.test_pars_annotations() - # File_operate_init.test_file_operators() - # File_operate_init.test_file_operate_one() - - # File_operate_init.test_converte_detect_records() - # File_operate_init.rename_xml_cls_name() - # File_operate_init.test_write_xml_with_boxandscore() - File_operate_init.metric_f1_score() \ No newline at end of file +from pathlib import Path + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + raise SystemExit( + subprocess.call( + [sys.executable, "-m", "pytest", str(root / "tests" / "test_core.py")], + cwd=root, + ) + ) diff --git a/test/Test_plot_tools.py b/test/Test_plot_tools.py index 98e56b4..8599afc 100644 --- a/test/Test_plot_tools.py +++ b/test/Test_plot_tools.py @@ -1,76 +1,19 @@ -# coding: utf-8 -import sys -sys.path.append('./') - -import imagedt -from imagedt.tools import plot_tools -from imagedt.tools import pdf - - -""" - Add: imagedt - - 1、git clone: ttps://github.com/hanranCode/ImageDT.git - git checkout dev - - 2、Add path - vi ~/.bashrc - export PYTHONPATH="/path_to_ImageDT:$PYTHONPATH" - - import imagedt -""" - - -class Test_PlotTools(object): - def __init__(self): - super(Test_PlotTools, self).__init__() - - def plot_datas_distribution(self, data_dir, title): - plot_tools.plot_datas(data_dir, title=title) +"""Legacy test entry point; runs the maintained Python 3 regression suite.""" - # 目标类统计 - Tese_label = [] - - import os - other_files = 0 - less_10_classes = 0 - less_10_others = 0 - taget_files = 0 - all_classes = os.listdir(data_dir) - others_cls = 0 - target_cls = 0 - - for class_name in all_classes: - class_files = imagedt.dir.loop(os.path.join(data_dir, class_name), ['.png', '.jpg']) - file_num = len(class_files) - if class_name.startswith('other'): - other_files += file_num - others_cls += 1 - else: - taget_files += file_num - target_cls += 1 - if file_num < 10: - less_10_classes +=1 - if class_name.startswith('other'): - less_10_others += 1 - - print "总images: ", other_files+taget_files - print "类别数: ", len(all_classes) - print "平均每类 images: ", (other_files+taget_files)/len(all_classes) - print "others 类别: ", others_cls, ' ', other_files, 'images' - print "目标类 类别: ", target_cls, ' ', taget_files, 'images' - print "少于10类的类别数: ", less_10_classes ,'(others {0} 类)'.format(less_10_others) - - - def test_pdf2image(sefl, pdf_path): - pdf.pdf2image(pdf_path, dpi=400) - - -if __name__ == '__main__': - plot_init = Test_PlotTools() - - data_dir = '/data2/李锦记/数据集/ljj_traindatas_191128_v2_check' - plot_init.plot_datas_distribution(data_dir, 'lijinji_train_20191128') - - # pdf_path = '/data/dataset/ocr/20181023/invoice/授权经销合范本.pdf' - # plot_init.test_pdf2image(pdf_path) \ No newline at end of file +import subprocess +import sys +from pathlib import Path + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + raise SystemExit( + subprocess.call( + [ + sys.executable, + "-m", + "pytest", + str(root / "tests" / "test_metrics_data_text.py"), + ], + cwd=root, + ) + ) diff --git a/test/Test_tensorflow.py b/test/Test_tensorflow.py index b703eb0..68d74fe 100644 --- a/test/Test_tensorflow.py +++ b/test/Test_tensorflow.py @@ -1,113 +1,19 @@ -# coding: utf-8 -import sys -sys.path.append('./') -import os -os.environ['CUDA_VISIBLE_DIVICES'] = '1' -import cv2 -import numpy as np -import tensorflow as tf -import nose.tools as ntools - -from imagedt.tensorflow.tools import DataInterface -from imagedt.tensorflow.tools import data_provider -from imagedt.tensorflow.tools import RecordWriter -import imagedt - -class Test_Own_Tensorflow(object): - def __init__(self): - super(Test_Own_Tensorflow, self).__init__() - self.data_instance = DataInterface.DataSets() - self.gen_datasets() - - def gen_datasets(self): - self.data_instance.train.set_x(np.zeros([32,10])) - self.data_instance.train.set_y(np.zeros([32])) - self.data_instance.test.set_x(np.ones([32, 10])) - self.data_instance.test.set_y(np.ones([32])) - - def test_get_next_batch_datas(self): - train_datas, train_labels = self.data_instance.train.next_batch(8, isFixed=False) - ntools.assert_equal(train_datas[0][0], 0) - ntools.assert_equal(train_labels[0], 0) - - test_datas, test_labels = self.data_instance.test.next_batch(8, isFixed=False) - ntools.assert_equal(test_datas[0][0], 1) - ntools.assert_equal(test_labels[0], 1) - - def test_data_dynamically_provider(self): - # train_file line: image [space] label - train_file = '/data/dataset/pg_one/PG/PG_data0726v2/train_datas/train_datasets_original/train.txt' - Data_Provider = data_provider.DataProvider(train_file, test_percent=0.01) - datas = Data_Provider.get_data_provider - - images, labels = datas.train.next_batch(32) - images, labels = data_provider.dynamically_loaded_data(images, labels) - - def write_class_tfrecords(self): - # write_cls_records(train_image_file, labels_file, save_dir) - # train.train_images_file: image.name label[split by ' '] - # labels_file: class_name label[split by '\t'] - train_images_file = '/data/dataset/heiren_yagao/训练数据/heiren_190121_datas/train_tf_infos_190123/train.txt' - labels_file = '/data/dataset/heiren_yagao/训练数据/heiren_190121_datas/train_tf_infos_190123/labels.txt' - save_dir = '/ssd_data/heiren_yagao/train_records' - RecordWriter.write_cls_records(train_images_file, labels_file, save_dir) +"""Legacy test entry point; runs the maintained Python 3 regression suite.""" - def write_detect_tfrecords(self): - # data_dir: contains {Jpg, Annotations} - data_dir = './test/sources/data_dir/' - save_dir = '/ssd_data/tmp' - RecordWriter.write_detect_records(data_dir, save_dir) - - - def noise_padd_op(self): - import random - img_file = imagedt.dir.loop('./test/sources/data_dir/', ['.jpg', '.png']) - cvmat = cv2.imread(img_file[0]) - with tf.Session() as sess: - for x in range(100): - set_image_size = random.randint(32, 680) - cvmat = cv2.resize(cvmat, (random.randint(223, 448), random.randint(223, 448))) - # cvmat = cv2.resize(cvmat, (set_image_size,set_image_size)) - noise_image = imagedt.tensorflow.tools.data_provider.tf_noise_padd(cvmat, max_edge=set_image_size, start_pixel=255) - image = sess.run(noise_image) - h, w, c = image.shape - print x, cvmat.shape, image.shape - assert h==w==set_image_size - - cv2.imwrite('/ssd_data/tmp/origin.png', cvmat) - cv2.imwrite('/ssd_data/tmp/noise_image.png', image) - - def write_inference_graph(self): - from imagedt.tensorflow.tools import write_pbmodel_summery - inference_pb_path = '/data/tmp/output/model_bk/mobilenet_v1_224_0.8122/frozen_inference_graph_skip.pb' - out_logdir = '/data/tmp/output/model_bk/mobilenet_v1_224_0.8122/inference_graph_log' - write_pbmodel_summery(inference_pb_path, out_logdir) - - def test_data_provider(self, data_dir): - data_dir = '/ssd_data/price_tag/train_records' - import time - get_batch_size = 128 - data_provider = imagedt.tensorflow.tools.data_provider.Data_Provider(data_dir) - train_data_batch = data_provider.read_from_tfrecord(batch_size=get_batch_size, image_shape=(96, 200, 1)) - with tf.Session() as sess: - coord = tf.train.Coordinator() - threads = tf.train.start_queue_runners(coord=coord) - try: - for _ in range(3): - st_time = time.time() - images, labels = sess.run(train_data_batch) - print "Fetch one batch {1} images, time cost: {0}".format(round(time.time()-st_time, 4), get_batch_size) - except: - coord.request_stop() - coord.join(threads) - - coord.request_stop() - coord.join(threads) - - -if __name__ == '__main__': - owntf_init = Test_Own_Tensorflow() - # owntf_init.write_class_tfrecords() - owntf_init.noise_padd_op() - # owntf_init.write_inference_graph() - # owntf_init.test_data_provider() \ No newline at end of file +import subprocess +import sys +from pathlib import Path + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + raise SystemExit( + subprocess.call( + [ + sys.executable, + "-m", + "pytest", + str(root / "tests" / "test_tensorflow.py"), + ], + cwd=root, + ) + ) diff --git a/test/Test_tools_metrics.py b/test/Test_tools_metrics.py index 30fda2b..8599afc 100644 --- a/test/Test_tools_metrics.py +++ b/test/Test_tools_metrics.py @@ -1,43 +1,19 @@ -# coding: utf-8 - -# coding: utf-8 -import imagedt - -class Test_Tools_Metrics(object): - def __init__(self): - super(Test_Tools_Metrics, self).__init__() - - def test_f1_score(self, predict_file, gt_file, labelt_file): - imagedt.tools.metrics.calculate_f1_score(predict_file, gt_file, labelt_file) - - def test_roc(self, predict_image_infos, show_fig=False): - imagedt.tools.metrics.roc(predict_image_infos, show_fig=show_fig) - - def test_generate_capt(self): - TG = imagedt.tools.text_generator.Text_Generator( - font_path='/data/project/idt-pitaya-serv-tensorflow/tf_utils/fonts/English_font', - langu='en', +"""Legacy test entry point; runs the maintained Python 3 regression suite.""" + +import subprocess +import sys +from pathlib import Path + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[1] + raise SystemExit( + subprocess.call( + [ + sys.executable, + "-m", + "pytest", + str(root / "tests" / "test_metrics_data_text.py"), + ], + cwd=root, ) - # font_type: Chinese_font ,English_font - generate_items = TG.gen_price_realtime_datas(batch_size=64) - print (generate_items) - - -if __name__ == '__main__': - Test_Metrics = Test_Tools_Metrics() - # gt_file line:[image name, predict class] 219549_0.jpg,723 - # predict_file line: [image name, predict class, confidence] 219549_0.jpg,723,0.999 - # labelt_file line: [object_id, cls_id, cls_name] :6973 0 维他奶维他柠檬茶250ml盒装_6973 - # gt_file = '/ssd_data/danon/meeting_shelfs/test_outinfos/label_infos.txt' - # predict_file = '/ssd_data/danon/meeting_shelfs/test_outinfos/test_infos.txt' - # labelt_file = '/ssd_data/danon/cls_records/labels_t.txt' - # # Test_Metrics.test_f1_score() - - # # predict_image_infos line format: Image_name,True_class_name,Label_id,Predict_id,Confidence,Match - # predict_image_infos = '/ssd_data/danon/meeting_shelfs/test_outinfos/image_classify_infos.csv' - # Test_Metrics.test_roc(predict_image_infos, show_fig=True) - - - Test_Metrics.test_generate_capt() - - + ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0e384ad --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +import os + +os.environ.setdefault("MPLBACKEND", "Agg") +os.environ.setdefault("MPLCONFIGDIR", "/tmp/imagedt-matplotlib") +os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2") +os.environ.setdefault("TF_NUM_INTEROP_THREADS", "1") +os.environ.setdefault("TF_NUM_INTRAOP_THREADS", "1") diff --git a/tests/fixtures/legacy_api.json b/tests/fixtures/legacy_api.json new file mode 100644 index 0000000..4fc6f19 --- /dev/null +++ b/tests/fixtures/legacy_api.json @@ -0,0 +1,1345 @@ +{ + "imagedt/converters/caffe_coreml.py": { + "convert": [ + "model_path", + "prototxt_path", + "label_path", + "red", + "green", + "blue", + "scale", + "bgr", + "output_model_name" + ] + }, + "imagedt/converters/caffe_tfmodel.py": { + "fatal_error": [ + "msg" + ], + "assert_exist_files": [], + "convert": [ + "prototxt_path", + "caffemodel_path", + "phase", + "output_model_name" + ] + }, + "imagedt/converters/taffe/caffe/resolver.py": { + "CaffeResolver": { + "import_caffe": [ + "self" + ], + "has_pycaffe": [ + "self" + ] + }, + "get_caffe_resolver": [], + "has_pycaffe": [], + "show_fallback_warning": [] + }, + "imagedt/converters/taffe/errors.py": { + "KaffeError": {}, + "print_stderr": [ + "msg" + ] + }, + "imagedt/converters/taffe/graph.py": { + "Node": { + "add_parent": [ + "self", + "parent_node" + ], + "add_child": [ + "self", + "child_node" + ], + "get_only_parent": [ + "self" + ], + "parameters": [ + "self" + ] + }, + "Graph": { + "add_node": [ + "self", + "node" + ], + "get_node": [ + "self", + "name" + ], + "get_input_nodes": [ + "self" + ], + "get_output_nodes": [ + "self" + ], + "topologically_sorted": [ + "self" + ], + "compute_output_shapes": [ + "self" + ], + "replaced": [ + "self", + "new_nodes" + ], + "transformed": [ + "self", + "transformers" + ] + }, + "GraphBuilder": { + "load": [ + "self" + ], + "filter_layers": [ + "self", + "layers" + ], + "make_node": [ + "self", + "layer" + ], + "make_input_nodes": [ + "self" + ], + "build": [ + "self" + ] + }, + "NodeMapper": { + "map": [ + "self" + ], + "map_chain": [ + "self", + "chain" + ], + "map_node": [ + "self", + "node" + ], + "commit": [ + "self", + "mapped_chains" + ] + } + }, + "imagedt/converters/taffe/layers.py": { + "NodeKind": { + "map_raw_kind": [ + "kind" + ], + "compute_output_shape": [ + "node" + ] + }, + "NodeDispatchError": {}, + "NodeDispatch": { + "get_handler_name": [ + "node_kind" + ], + "get_handler": [ + "self", + "node_kind", + "prefix" + ] + }, + "LayerAdapter": { + "parameters": [ + "self" + ], + "get_kernel_value": [ + "scalar", + "repeated", + "idx", + "default" + ], + "kernel_parameters": [ + "self" + ] + } + }, + "imagedt/converters/taffe/shapes.py": { + "get_filter_output_shape": [ + "i_h", + "i_w", + "params", + "round_func" + ], + "get_strided_kernel_output_shape": [ + "node", + "round_func" + ], + "shape_not_implemented": [ + "node" + ], + "shape_identity": [ + "node" + ], + "shape_scalar": [ + "node" + ], + "shape_data": [ + "node" + ], + "shape_mem_data": [ + "node" + ], + "shape_concat": [ + "node" + ], + "shape_convolution": [ + "node" + ], + "shape_pool": [ + "node" + ], + "shape_inner_product": [ + "node" + ] + }, + "imagedt/converters/taffe/tensorflow/network.py": { + "layer": [ + "op" + ], + "Network": { + "setup": [ + "self" + ], + "load": [ + "self", + "data_path", + "session", + "ignore_missing" + ], + "feed": [ + "self" + ], + "get_output": [ + "self" + ], + "get_unique_name": [ + "self", + "prefix" + ], + "make_var": [ + "self", + "name", + "shape" + ], + "validate_padding": [ + "self", + "padding" + ], + "conv": [ + "self", + "input", + "k_h", + "k_w", + "c_o", + "s_h", + "s_w", + "name", + "relu", + "padding", + "group", + "biased" + ], + "relu": [ + "self", + "input", + "name" + ], + "max_pool": [ + "self", + "input", + "k_h", + "k_w", + "s_h", + "s_w", + "name", + "padding" + ], + "avg_pool": [ + "self", + "input", + "k_h", + "k_w", + "s_h", + "s_w", + "name", + "padding" + ], + "lrn": [ + "self", + "input", + "radius", + "alpha", + "beta", + "name", + "bias" + ], + "concat": [ + "self", + "inputs", + "axis", + "name" + ], + "add": [ + "self", + "inputs", + "name" + ], + "fc": [ + "self", + "input", + "num_out", + "name", + "relu" + ], + "softmax": [ + "self", + "input", + "name" + ], + "batch_normalization": [ + "self", + "input", + "name", + "scale_offset", + "relu" + ], + "dropout": [ + "self", + "input", + "keep_prob", + "name" + ] + } + }, + "imagedt/converters/taffe/tensorflow/transformer.py": { + "get_padding_type": [ + "kernel_params", + "input_shape", + "output_shape" + ], + "TensorFlowNode": { + "format": [ + "self", + "arg" + ], + "pair": [ + "self", + "key", + "value" + ], + "emit": [ + "self" + ] + }, + "MaybeActivated": {}, + "TensorFlowMapper": { + "get_kernel_params": [ + "self", + "node" + ], + "map_convolution": [ + "self", + "node" + ], + "map_relu": [ + "self", + "node" + ], + "map_pooling": [ + "self", + "node" + ], + "map_inner_product": [ + "self", + "node" + ], + "map_softmax": [ + "self", + "node" + ], + "map_lrn": [ + "self", + "node" + ], + "map_concat": [ + "self", + "node" + ], + "map_dropout": [ + "self", + "node" + ], + "map_batch_norm": [ + "self", + "node" + ], + "map_eltwise": [ + "self", + "node" + ], + "commit": [ + "self", + "chains" + ] + }, + "TensorFlowEmitter": { + "indent": [ + "self" + ], + "outdent": [ + "self" + ], + "statement": [ + "self", + "s" + ], + "emit_imports": [ + "self" + ], + "emit_class_def": [ + "self", + "name" + ], + "emit_setup_def": [ + "self" + ], + "emit_parents": [ + "self", + "chain" + ], + "emit_node": [ + "self", + "node" + ], + "emit": [ + "self", + "name", + "chains" + ] + }, + "TensorFlowTransformer": { + "load": [ + "self", + "def_path", + "data_path", + "phase" + ], + "transform_data": [ + "self" + ], + "transform_source": [ + "self" + ] + } + }, + "imagedt/converters/taffe/transformers.py": { + "DataInjector": { + "load": [ + "self" + ], + "load_using_caffe": [ + "self" + ], + "load_using_pb": [ + "self" + ], + "normalize_pb_data": [ + "self", + "layer" + ], + "adjust_parameters": [ + "self", + "node", + "data" + ] + }, + "DataReshaper": { + "has_spatial_parent": [ + "self", + "node" + ], + "map": [ + "self", + "node_kind" + ] + }, + "SubNodeFuser": { + "is_eligible_pair": [ + "self", + "parent", + "child" + ], + "merge": [ + "self", + "parent", + "child" + ] + }, + "ReLUFuser": { + "is_eligible_pair": [ + "self", + "parent", + "child" + ], + "merge": [ + "self", + "parent", + "_" + ] + }, + "BatchNormScaleBiasFuser": { + "is_eligible_pair": [ + "self", + "parent", + "child" + ], + "merge": [ + "self", + "parent", + "child" + ] + }, + "BatchNormPreprocessor": {}, + "NodeRenamer": {}, + "ParameterNamer": {} + }, + "imagedt/converters/tfmodel_lite.py": { + "convert": [ + "graph_def_file", + "output_file", + "output_format", + "inference_type", + "inference_input_type", + "input_array", + "input_shape", + "output_arrays" + ] + }, + "imagedt/converters/tfrecords.py": { + "int64_feature": [ + "values" + ], + "bytes_feature": [ + "values" + ], + "float_feature": [ + "values" + ], + "ImageReader": { + "read_image_dims": [ + "self", + "sess", + "image_data" + ], + "decode_jpeg": [ + "self", + "sess", + "image_data" + ] + }, + "get_files_dir_dict": [ + "all_files" + ], + "get_image_pairs": [ + "data_dir" + ], + "get_label_dict": [ + "dir_dict" + ], + "write_laebl_lines": [ + "root_dir", + "all_files", + "label_map" + ], + "image_to_tfexample": [ + "image_data", + "image_format", + "height", + "width", + "class_id" + ], + "convert_to_tfrecord": [ + "f_lines", + "root_dir" + ] + }, + "imagedt/decorator/decorator_time.py": { + "time_cost": [ + "func" + ] + }, + "imagedt/dir/dir_loop.py": { + "LoopFile": { + "start": [ + "self", + "func" + ], + "loop_file": [ + "self", + "root_dir" + ] + }, + "loop": [ + "root", + "extensions" + ], + "get_dir_images": [ + "root", + "extensions" + ] + }, + "imagedt/file/file_operate.py": { + "FilesOp": { + "get_abs_path": [ + "self", + "file_dir", + "files_list" + ], + "check_dir_exist": [ + "self", + "path" + ], + "check_file_exist": [ + "self", + "fpath" + ], + "fillter_dirname": [ + "self", + "file_paths" + ], + "fillter_file_type": [ + "self", + "file_paths" + ], + "rename_with_uuid": [ + "self", + "fname", + "uid" + ], + "rename_data_pairs": [ + "self", + "file_pair", + "uid" + ], + "assert_not_in_op": [ + "self", + "member", + "container" + ], + "assert_in_op": [ + "self", + "member", + "container" + ], + "assert_instance_op": [ + "self", + "inst", + "tar_inst" + ], + "del_broken_image": [ + "self", + "root_dir" + ], + "check_data_pairs": [ + "self", + "root_dir", + "anno_type" + ], + "get_jpg_xml_pairs": [ + "self", + "root_dir", + "anno_type" + ], + "rename_file_with_uuid": [ + "self", + "root_dir" + ], + "rename_single_file_uuid": [ + "self", + "root_dir" + ], + "loop_dir": [ + "self", + "root_dir" + ], + "rename_class_dir": [ + "self", + "root_dir", + "before_cls_str" + ] + } + }, + "imagedt/file/file_write.py": { + "write_txt": [ + "data", + "path", + "separator" + ], + "write_csv": [ + "data", + "path", + "is_excel" + ], + "read_csv": [ + "path", + "delimiter" + ], + "readlines": [ + "file_path" + ], + "write_json": [ + "json_objects", + "file_path" + ], + "read_json": [ + "file_path" + ], + "decode_strs": [ + "strs" + ], + "writeExcel": [ + "write_infos_mul", + "excel_path", + "sheetNames", + "set_color" + ] + }, + "imagedt/file/parse_annotation.py": { + "Anno_OP": { + "read_xml": [ + "self", + "fxml" + ], + "parse_bndx_object": [ + "self", + "bndbox_object" + ], + "read_txt_bndbox": [ + "self", + "fpath" + ], + "get_object_bndbox": [ + "self", + "sku_object" + ], + "get_label_object": [ + "self", + "fxml" + ], + "parse_lab_xml": [ + "self", + "fxml" + ], + "load_annoataion": [ + "self", + "poly_file" + ], + "parse_poly_locations": [ + "self", + "loca_info" + ], + "convert_poly_to_Xml": [ + "self", + "poly_file" + ], + "reset_xml_cls": [ + "self", + "xml_dir", + "name", + "desc", + "resetid_list" + ], + "filter_xml_classes": [ + "self", + "xml_dir", + "name", + "desc", + "resetid_list" + ], + "write_anno_object": [ + "self", + "annotation", + "xmlname" + ], + "wirte_xml_file": [ + "self", + "xml_etree", + "xml_path" + ], + "write_xml": [ + "self", + "xmlname", + "bndboxs", + "scores", + "classes", + "thresh" + ] + } + }, + "imagedt/image/image_pil.py": { + "alpha_to_color": [ + "image", + "color" + ] + }, + "imagedt/image/process.py": { + "clear_dir": [ + "images_dir", + "debug", + "multi_process" + ], + "noise_padd": [ + "img", + "edge_size", + "start_pixel_value", + "end_pixel_value" + ], + "remove_broken_image": [ + "data_dir" + ], + "resize_with_scale": [ + "cvmat", + "max_length" + ], + "padd_pixel": [ + "img", + "edge_size_w", + "edge_size_h" + ], + "reduce_imagenet_mean": [ + "cv_mat" + ], + "swap_chanel_to_RGB": [ + "cvmat" + ], + "vgg_preprocesing": [ + "cvmat" + ], + "inception_preprocesing": [ + "cvmat" + ], + "our_processing": [ + "cvmat" + ], + "md5": [ + "fname" + ], + "get_box": [ + "bndbox", + "loca_type" + ], + "put_cvtext": [ + "img", + "text", + "posi", + "color", + "fontScale", + "thickness", + "font_type" + ], + "draw_cvrect": [ + "img", + "bndboxes", + "box_color", + "border", + "font_color", + "draw_strs", + "fontScale", + "compare_strs", + "draw_location", + "font_type", + "thickness" + ], + "cv_show": [ + "imgs", + "win_name" + ], + "resize_dir_images_with_scale": [ + "image_dir", + "resize_max_edge" + ], + "points_to_contour": [ + "points" + ], + "points_to_contours": [ + "points" + ], + "put_text": [ + "img", + "text", + "pos", + "scale", + "color", + "thickness" + ], + "draw_contours": [ + "img", + "contours", + "idx", + "color", + "border_width" + ], + "polly_points2contours": [ + "points" + ], + "rename_imfiles_with_md5": [ + "imdir", + "rename_xml" + ] + }, + "imagedt/tensorflow/lite/lite_wrapper.py": { + "Lite_Wrapper": { + "load_model": [ + "self" + ], + "predict": [ + "self", + "input_data" + ] + } + }, + "imagedt/tensorflow/network/lenet.py": { + "lenet": [ + "images", + "num_classes", + "is_training", + "drop_keep_prob", + "prediction_fn", + "scope" + ] + }, + "imagedt/tensorflow/optim/loss.py": { + "smooth_l1_loss": [ + "y_true", + "y_pred" + ], + "arcface_loss": [ + "embedding", + "labels", + "out_num", + "w_init", + "s", + "m" + ] + }, + "imagedt/tensorflow/tools/DataInterface.py": { + "DataSet": { + "x": [ + "self" + ], + "y": [ + "self" + ], + "count": [ + "self" + ], + "set_x": [ + "self", + "x" + ], + "set_y": [ + "self", + "y" + ], + "reset_step": [ + "self" + ], + "batch_count": [ + "self", + "batch_size", + "isFixed" + ], + "next_batch": [ + "self", + "batch_size", + "isFixed" + ] + }, + "DataSets": { + "test": [ + "self" + ], + "validate": [ + "self" + ], + "train": [ + "self" + ], + "x_shape": [ + "self" + ], + "set_x_shape": [ + "self", + "x_shape" + ], + "y_shape": [ + "self" + ], + "set_y_shape": [ + "self", + "y_shape" + ] + } + }, + "imagedt/tensorflow/tools/Records.py": { + "ImageReader": { + "read_image_dims": [ + "self", + "sess", + "image_data" + ], + "decode_jpeg": [ + "self", + "sess", + "image_data" + ] + }, + "Record_Writer": { + "image_to_tfexample": [ + "self", + "image_data", + "image_format", + "height", + "width", + "class_id" + ], + "convert_to_tfrecord": [ + "self", + "f_lines", + "svae_dir", + "dataset_type" + ], + "map_int": [ + "self", + "m_list" + ], + "create_tf_example": [ + "self", + "sess", + "jpg_path", + "anno_infos" + ], + "write_cls_records": [ + "self", + "train_images_file", + "save_dir", + "test_ratio" + ], + "converte_anno_info": [ + "self", + "data_dir", + "det_cls_name" + ], + "write_detect_records": [ + "self", + "data_dir", + "save_dir", + "record_name" + ] + }, + "write_pbmodel_summery": [ + "tf_pbmodel", + "log_dir" + ] + }, + "imagedt/tensorflow/tools/data_provider.py": { + "dynamically_loaded_data": [ + "image_paths", + "labels", + "height", + "width" + ], + "resize_image_keep_aspect": [ + "image", + "max_edge" + ], + "tf_noise_padd": [ + "images", + "max_edge", + "start_pixel", + "end_pixel" + ], + "tf_JitterCut": [ + "images", + "jitter" + ], + "Data_Provider": { + "read_from_tfrecord": [ + "self", + "batch_size", + "image_shape" + ] + } + }, + "imagedt/tensorflow/tools/dataset_util.py": { + "int64_list_feature": [ + "value" + ], + "bytes_list_feature": [ + "value" + ], + "float_list_feature": [ + "value" + ], + "int64_feature": [ + "values" + ], + "bytes_feature": [ + "values" + ], + "float_feature": [ + "values" + ], + "read_examples_list": [ + "path" + ], + "recursive_parse_xml_to_dict": [ + "xml" + ], + "make_initializable_iterator": [ + "dataset" + ], + "read_dataset": [ + "file_read_func", + "decode_func", + "input_files", + "config" + ] + }, + "imagedt/tensorflow/tools/tfmodel_wrapper.py": { + "TFmodel_Wrapper": { + "reset_node_name": [ + "self" + ], + "predict": [ + "self", + "images" + ], + "extract": [ + "self", + "image" + ] + } + }, + "imagedt/tensorflow/tools/tfrecords.py": { + "visual_records": [ + "tfrecords_path" + ] + }, + "imagedt/tools/com_tools.py": { + "getip": [] + }, + "imagedt/tools/metrics/detect_eval.py": { + "parse_rec": [ + "filename" + ], + "parse_detect": [ + "detannos" + ], + "load_detect_lines": [ + "detannos", + "conf" + ], + "get_xmls_basename": [ + "xml_path" + ], + "voc_eval": [ + "detpath", + "annopath", + "ovthresh", + "det_file_type", + "conf", + "set_gt_labels" + ] + }, + "imagedt/tools/metrics/f1_score.py": { + "load_label_Dict": [ + "lines" + ], + "write_predict_details": [ + "predict_infos", + "label_infos", + "label_map_dict" + ], + "gernerate_infos_dict": [ + "predicts", + "true_labels", + "label_map_dict" + ], + "calculate_f1_score": [ + "predict_file", + "gt_file", + "labelt_file" + ] + }, + "imagedt/tools/metrics/roc_curve.py": { + "roc": [ + "test_file", + "show_fig" + ] + }, + "imagedt/tools/pdf.py": { + "pdf2image": [ + "pdf_path", + "save_out", + "dpi" + ] + }, + "imagedt/tools/plot_curve.py": { + "PlotCurve": { + "load_dataset_infos": [ + "self", + "data_dir" + ], + "set_xy_values": [ + "self", + "sample_dict" + ], + "add_bar_values": [ + "self", + "rects" + ], + "add_avg_yline": [ + "self" + ], + "plot_datas": [ + "self", + "data_dir", + "add_bar_values", + "title", + "curve_type" + ] + } + }, + "imagedt/tools/text_generator/background_generator.py": { + "BackgroundGenerator": { + "gaussian_noise": [ + "cls", + "height", + "width" + ], + "plain_white": [ + "cls", + "height", + "width" + ], + "quasicrystal": [ + "cls", + "height", + "width" + ], + "picture": [ + "cls", + "height", + "width" + ] + } + }, + "imagedt/tools/text_generator/computer_text_generator.py": { + "ComputerTextGenerator": { + "generate": [ + "cls", + "text", + "font", + "text_color", + "font_size", + "orientation", + "space_width" + ] + } + }, + "imagedt/tools/text_generator/data_generator.py": { + "FakeTextDataGenerator": { + "generate_from_tuple": [ + "cls", + "t" + ], + "generate": [ + "cls", + "index", + "text", + "font", + "out_dir", + "size", + "extension", + "skewing_angle", + "random_skew", + "blur", + "random_blur", + "background_type", + "distorsion_type", + "distorsion_orientation", + "is_handwritten", + "name_format", + "width", + "alignment", + "text_color", + "orientation", + "space_width", + "save" + ] + } + }, + "imagedt/tools/text_generator/distorsion_generator.py": { + "DistorsionGenerator": { + "apply_func_distorsion": [ + "cls", + "image", + "vertical", + "horizontal", + "max_offset", + "func" + ], + "sin": [ + "cls", + "image", + "vertical", + "horizontal" + ], + "cos": [ + "cls", + "image", + "vertical", + "horizontal" + ], + "random": [ + "cls", + "image", + "vertical", + "horizontal" + ] + } + }, + "imagedt/tools/text_generator/handwritten_text_generator.py": { + "HandwrittenTextGenerator": { + "generate": [ + "cls", + "text" + ] + } + }, + "imagedt/tools/text_generator/text_generator.py": { + "Text_Generator": { + "set_params": [ + "self", + "tests" + ], + "generate_texts": [ + "self" + ], + "gen_price_realtime_datas": [ + "self", + "batch_size" + ] + } + } +} diff --git a/tests/test_caffe_backends.py b/tests/test_caffe_backends.py new file mode 100644 index 0000000..9a313c4 --- /dev/null +++ b/tests/test_caffe_backends.py @@ -0,0 +1,317 @@ +from types import SimpleNamespace + +import numpy as np +import pytest +from google.protobuf import text_format + +from imagedt.converters.taffe.caffe import caffe_pb2 +from imagedt.converters.taffe.errors import KaffeError +from imagedt.converters.taffe.graph import Graph, GraphBuilder, Node +from imagedt.converters.taffe.layers import ( + KernelParameters, + LayerAdapter, + NodeDispatch, + NodeKind, +) +from imagedt.converters.taffe.shapes import ( + TensorShape, + get_filter_output_shape, + shape_not_implemented, + shape_scalar, +) +from imagedt.converters.taffe.transformers import ( + DataInjector, + DataReshaper, + NodeRenamer, + ParameterNamer, + ReLUFuser, +) + + +def test_protobuf_roundtrip_and_shape(): + net = caffe_pb2.NetParameter(name="Tiny") + text_format.Merge('input: "data" input_shape { dim: 1 dim: 3 dim: 4 dim: 4 }', net) + assert caffe_pb2.NetParameter.FromString(net.SerializeToString()) == net + assert get_filter_output_shape( + 10, 10, KernelParameters(3, 3, 2, 2, 0, 0), np.floor + ) == (4, 4) + assert shape_scalar(None) == TensorShape(1, 1, 1, 1) + with pytest.raises(NotImplementedError): + shape_not_implemented(None) + assert NodeDispatch.get_handler_name("InnerProduct") == "inner_product" + assert LayerAdapter.get_kernel_value(0, [3], 0) == 3 + assert LayerAdapter.get_kernel_value(0, [], 0, 1) == 1 + with pytest.raises(ValueError): + LayerAdapter.get_kernel_value(0, [], 0) + + +def test_graph_sort_cycle_and_rename(): + first, second = Node("a", NodeKind.Data), Node("b", NodeKind.ReLU) + second.add_parent(first) + graph = Graph([second, first], name="Tiny") + assert graph.topologically_sorted() == [first, second] + assert graph.get_input_nodes() == [first] and graph.get_output_nodes() == [second] + NodeRenamer(lambda node: "new_" + node.name)(graph) + assert graph.get_node("new_a") is first + with pytest.raises(KaffeError): + graph.get_node("missing") + first.add_parent(second) + with pytest.raises(KaffeError): + graph.topologically_sorted() + + +def test_large_graph_without_recursion(): + nodes = [Node(str(i), NodeKind.ReLU) for i in range(2000)] + for parent, child in zip(nodes, nodes[1:], strict=False): + child.add_parent(parent) + assert Graph(nodes).topologically_sorted() == nodes + + +def test_relu_fuse_and_data_reshape(): + layer = caffe_pb2.LayerParameter(name="conv", type="Convolution") + layer.convolution_param.bias_term = True + parent = Node("conv", NodeKind.Convolution, layer) + relu = Node("relu", NodeKind.ReLU) + relu.add_parent(parent) + child = Node("next", NodeKind.Softmax) + child.add_parent(relu) + parent.data = [np.ones((2, 3, 1, 1)), np.zeros(2)] + graph = ReLUFuser()(Graph([parent, relu, child])) + assert graph.nodes == [parent, child] and child.parents == [parent] + assert parent.metadata["relu"] + DataReshaper({NodeKind.Convolution: (2, 3, 1, 0)})(graph) + assert parent.data[0].shape == (1, 1, 3, 2) + ParameterNamer()(graph) + assert set(parent.data) == {"weights", "biases"} + + +def make_model(tmp_path): + definition = tmp_path / "tiny.prototxt" + definition.write_text("""name: "Tiny" +input: "data" +input_shape { dim: 1 dim: 3 dim: 2 dim: 2 } +layer { name: "score" type: "InnerProduct" bottom: "data" top: "score" inner_product_param { num_output: 2 } } +layer { name: "prob" type: "Softmax" bottom: "score" top: "prob" } +""") + weights = caffe_pb2.NetParameter(name="Tiny") + layer = weights.layer.add(name="score", type="InnerProduct") + blob = layer.blobs.add() + blob.shape.dim.extend([2, 12]) + blob.data.extend([0.0] * 24) + bias = layer.blobs.add() + bias.shape.dim.extend([2]) + bias.data.extend([0.0, 2.0]) + model = tmp_path / "tiny.caffemodel" + model.write_bytes(weights.SerializeToString()) + return definition, model + + +def test_graph_builder_and_weights(tmp_path): + definition, model = make_model(tmp_path) + graph = GraphBuilder(definition).build() + assert graph.get_node("score").output_shape == TensorShape(1, 2, 1, 1) + injector = DataInjector(definition, model) + injector(graph) + assert graph.get_node("score").data[0].shape == (2, 12) + assert "score" in str(graph) + + +@pytest.mark.tensorflow +@pytest.mark.integration +def test_caffe_to_frozen_graph(tmp_path): + pytest.importorskip("tensorflow") + from imagedt.converters.caffe_tfmodel import convert + from imagedt.tensorflow.tools.tfmodel_wrapper import TFmodel_Wrapper + + definition, model = make_model(tmp_path) + convert(str(definition), str(model)) + with TFmodel_Wrapper(tmp_path / "freeze_graph_model.pb", "data", "prob") as wrapper: + output = wrapper.predict(np.zeros((1, 2, 2, 3), np.float32)) + assert output[0]["class"] == 1 and output[0]["confidence"] > 0.8 + + +def test_coreml_removed_backend(monkeypatch): + from imagedt.converters import caffe_coreml + + with pytest.raises(NotImplementedError, match="Caffe"): + caffe_coreml.convert("model", "definition", "labels") + + +def test_optional_dependency_diagnostic(monkeypatch): + from imagedt import _backends + + def missing(name): + raise ModuleNotFoundError("missing", name=name) + + monkeypatch.setattr(_backends, "import_module", missing) + with pytest.raises(ImportError, match=r"imagedt\[tensorflow\]"): + _backends.tensorflow_v1() + + +def test_caffe_shape_layer_variants(tmp_path): + definition = tmp_path / "conv.prototxt" + definition.write_text("""name: "ConvTiny" +layer { name: "data" type: "Input" top: "data" input_param { shape { dim: 1 dim: 4 dim: 8 dim: 8 } } } +layer { name: "conv" type: "Convolution" bottom: "data" top: "conv" convolution_param { num_output: 4 kernel_size: 3 pad: 1 group: 2 } } +layer { name: "pool" type: "Pooling" bottom: "conv" top: "pool" pooling_param { pool: MAX kernel_size: 2 stride: 2 } } +layer { name: "pool2" type: "Pooling" bottom: "conv" top: "pool2" pooling_param { pool: MAX kernel_size: 2 stride: 2 } } +layer { name: "cat" type: "Concat" bottom: "pool" bottom: "pool2" top: "cat" concat_param { axis: 1 } } +""") + graph = GraphBuilder(definition).build() + assert graph.get_node("data").output_shape == TensorShape(1, 4, 8, 8) + assert graph.get_node("conv").output_shape == TensorShape(1, 4, 8, 8) + assert graph.get_node("pool").output_shape == TensorShape(1, 4, 4, 4) + assert graph.get_node("cat").output_shape.channels == 8 + # Filter excludes the specified phase, independent of include rules. + builder = GraphBuilder(definition) + layer = caffe_pb2.LayerParameter(name="excluded", type="ReLU") + layer.exclude.add(phase=caffe_pb2.TEST) + assert builder.filter_layers([layer]) == [] + + +def test_transformer_source_quotes_and_padding(): + pytest.importorskip("tensorflow") + from imagedt.converters.taffe.tensorflow.transformer import ( + TensorFlowEmitter, + TensorFlowNode, + get_padding_type, + ) + + node = TensorFlowNode("feed", "name'withquote", axis=3) + node.node = SimpleNamespace(name="safe") + assert "\\'" in node.emit() or '"name\'withquote"' in node.emit() + assert ( + get_padding_type( + KernelParameters(3, 3, 1, 1, 0, 0), + TensorShape(1, 3, 8, 8), + TensorShape(1, 4, 6, 6), + ) + == "VALID" + ) + assert ( + get_padding_type( + KernelParameters(3, 3, 1, 1, 1, 1), + TensorShape(1, 3, 8, 8), + TensorShape(1, 4, 8, 8), + ) + == "SAME" + ) + with pytest.raises(KaffeError): + TensorFlowEmitter().emit("invalid-name", []) + + +def test_batchnorm_stats_and_kernel_validation(): + from imagedt.converters.taffe.transformers import ( + BatchNormPreprocessor, + BatchNormScaleBiasFuser, + ) + + layer = caffe_pb2.LayerParameter(name="bn", type="BatchNorm") + node = Node("bn", NodeKind.BatchNorm, layer) + node.data = [np.ones(3), np.ones(3) * 2, np.array(2.0)] + graph = BatchNormPreprocessor()(Graph([node])) + np.testing.assert_allclose(node.data[0], 0.5) + np.testing.assert_allclose(node.data[1], 1.0) + scale_layer = caffe_pb2.LayerParameter(name="scale", type="Scale") + scale_layer.scale_param.bias_term = True + scale = Node("scale", NodeKind.Scale, scale_layer) + scale.data = [np.ones(3), np.zeros(3)] + scale.add_parent(node) + graph = BatchNormScaleBiasFuser()(Graph([node, scale])) + assert graph.nodes == [node] and node.scale_bias_node is scale + with pytest.raises(KaffeError): + Graph([node]).transformed([lambda _: None]) + with pytest.raises(KaffeError): + node.get_only_parent() + assert NodeKind.map_raw_kind("invalid") is None + + +@pytest.mark.tensorflow +@pytest.mark.integration +def test_terminal_fc_and_serialized_batchnorm(tmp_path): + # Test value: protects=Caffe terminal-layer output and serialized BN scaling; + # failure=wrong op name/4D stats; oracle=analytic normalized scores; seam=none. + pytest.importorskip("tensorflow") + from imagedt.converters.caffe_tfmodel import convert + from imagedt.tensorflow.tools.tfmodel_wrapper import TFmodel_Wrapper + + definition, model = make_model(tmp_path) + definition.write_text(definition.read_text().split('layer { name: "prob"')[0]) + weights = caffe_pb2.NetParameter.FromString(model.read_bytes()) + # Nonzero FC weights also expose flatten/channel-order errors. + weights.layer[0].blobs[0].data[:] = np.arange(24, dtype=np.float32).tolist() + model.write_bytes(weights.SerializeToString()) + data = np.arange(12, dtype=np.float32).reshape(1, 2, 2, 3) + expected = data.transpose(0, 3, 1, 2).reshape(1, 12) @ np.arange( + 24, dtype=np.float32 + ).reshape(2, 12).T + [0, 2] + convert(str(definition), str(model), output_model_name="fc.pb") + with TFmodel_Wrapper(tmp_path / "fc.pb", "data", "score") as wrapper: + actual = wrapper.sess.run( + wrapper.tensor_dict[wrapper.output_node[0]], {wrapper.image_tensor: data} + ) + np.testing.assert_allclose(actual, expected) + definition.write_text( + definition.read_text() + + 'layer { name: "bn" type: "BatchNorm" bottom: "score" top: "bn" }\n' + ) + layer = weights.layer.add(name="bn", type="BatchNorm") + for values in ([2.0, 4.0], [8.0, 8.0], [2.0]): + blob = layer.blobs.add() + blob.shape.dim.extend([len(values)]) + blob.data.extend(values) + model.write_bytes(weights.SerializeToString()) + convert(str(definition), str(model), output_model_name="bn.pb") + with TFmodel_Wrapper(tmp_path / "bn.pb", "data", "bn") as wrapper: + actual = wrapper.sess.run( + wrapper.tensor_dict[wrapper.output_node[0]], {wrapper.image_tensor: data} + ) + np.testing.assert_allclose( + actual, (expected - [1, 2]) / np.sqrt(4.0 + 1e-5), rtol=1e-5 + ) + + definition.write_text( + definition.read_text() + + 'layer { name: "scale" type: "Scale" bottom: "bn" top: "scale" ' + + "scale_param { bias_term: true } }\n" + ) + layer = weights.layer.add(name="scale", type="Scale") + for values in ([2.0, 3.0], [-1.0, 5.0]): + blob = layer.blobs.add() + blob.shape.dim.extend([len(values)]) + blob.data.extend(values) + model.write_bytes(weights.SerializeToString()) + convert(str(definition), str(model), output_model_name="scale.pb") + with TFmodel_Wrapper(tmp_path / "scale.pb", "data", "bn") as wrapper: + actual = wrapper.sess.run( + wrapper.tensor_dict[wrapper.output_node[0]], {wrapper.image_tensor: data} + ) + np.testing.assert_allclose( + actual, + (expected - [1, 2]) / np.sqrt(4.0 + 1e-5) * [2, 3] + [-1, 5], + rtol=1e-5, + ) + + +def test_caffe_padding_rejects_non_equivalent_same(): + # Test value: protects=conversion fidelity for even inputs and stride two; + # failure=equal output shape hides asymmetric TF padding; oracle=explicit rejection; seam=none. + pytest.importorskip("tensorflow") + from imagedt.converters.taffe.tensorflow.transformer import get_padding_type + + assert ( + get_padding_type( + KernelParameters(3, 3, 2, 2, 1, 1), + TensorShape(1, 1, 4, 4), + TensorShape(1, 1, 2, 2), + ) + is None + ) + assert ( + get_padding_type( + KernelParameters(3, 3, 2, 2, 1, 1), + TensorShape(1, 1, 5, 5), + TensorShape(1, 1, 3, 3), + ) + == "SAME" + ) diff --git a/tests/test_contracts.py b/tests/test_contracts.py new file mode 100644 index 0000000..6d18bbe --- /dev/null +++ b/tests/test_contracts.py @@ -0,0 +1,76 @@ +"""Guard the original public source interfaces and all installable module paths.""" + +import ast +import importlib +import inspect +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +API = json.loads( + (ROOT / "tests" / "fixtures" / "legacy_api.json").read_text(encoding="utf-8") +) + + +@pytest.mark.parametrize("path,definitions", list(API.items()), ids=list(API)) +def test_legacy_function_names_and_arguments(path, definitions): + tree = ast.parse((ROOT / path).read_text(encoding="utf-8")) + nodes = { + node.name: node + for node in tree.body + if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + } + for name, contract in definitions.items(): + if name not in nodes: + # Consolidated implementations may be re-exported at the old path. + pytest.importorskip("tensorflow") + module = importlib.import_module(path[:-3].replace("/", ".")) + value = getattr(module, name) + if isinstance(contract, list): + assert list(inspect.signature(value).parameters) == contract + else: + for method, arguments in contract.items(): + assert ( + list(inspect.signature(getattr(value, method)).parameters) + == arguments + ) + continue + node = nodes[name] + if isinstance(contract, list): + assert [arg.arg for arg in node.args.args] == contract + else: + methods = { + method.name: method + for method in node.body + if isinstance(method, ast.FunctionDef) + } + for method, arguments in contract.items(): + assert method in methods, f"{path}: removed {name}.{method}" + assert [arg.arg for arg in methods[method].args.args] == arguments + + +@pytest.mark.parametrize( + "module", + [ + path.relative_to(ROOT) + .with_suffix("") + .as_posix() + .replace("/", ".") + .removesuffix(".__init__") + for path in (ROOT / "imagedt").rglob("*.py") + if path.name != "test_lenet.py" + ], +) +def test_module_imports(module): + if ".tensorflow" in module or module in ( + "imagedt.converters.caffe_tfmodel", + "imagedt.converters.tfrecords", + ): + pytest.importorskip("tensorflow") + if ".taffe" in module: + pytest.importorskip("google.protobuf") + if module in ("imagedt.tools.plot_curve", "imagedt.tools.metrics.roc_curve"): + pytest.importorskip("matplotlib") + importlib.import_module(module) diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..f85bb7a --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,223 @@ +import hashlib +import subprocess +import sys +from pathlib import Path + +import cv2 +import numpy as np +import pytest +from PIL import Image + +import imagedt +from imagedt.dir import LoopFile, get_dir_images, loop +from imagedt.file import Anno_OP, FilesOp, file_write +from imagedt.image import process + + +def test_import_is_lightweight(): + subprocess.run( + [ + sys.executable, + "-c", + "import imagedt, sys; assert not {'tensorflow', 'matplotlib', 'cv2', 'xlwt'} & sys.modules.keys(); assert imagedt.__version__ == '0.6.0rc1'", + ], + check=True, + ) + assert imagedt.FilesOp is FilesOp + assert imagedt.Anno_OP is Anno_OP + assert imagedt.noise_padd is process.noise_padd + assert "write_txt" in dir(imagedt) + with pytest.raises(AttributeError): + _ = imagedt.no_such_attribute + + +@pytest.mark.parametrize( + "data", ["你好", ["你好", "world"], [["a", 2]], {"名字": "值"}] +) +def test_write_txt(tmp_path, data): + path = tmp_path / "text.txt" + file_write.write_txt(data, path) + assert path.read_text(encoding="utf-8") + + +def test_csv_json_unicode(tmp_path): + rows = [["中文", 'comma,quote"\nline'], ["1", "2"]] + path = tmp_path / "data.csv" + file_write.write_csv(rows, path, is_excel=True) + assert path.read_bytes().startswith(b"\xef\xbb\xbf") + assert file_write.read_csv(path) == rows + file_write.write_json({"中文": [1, 2]}, tmp_path / "data.json") + assert file_write.read_json(tmp_path / "data.json") == {"中文": [1, 2]} + assert file_write.decode_strs("中文".encode()) == "中文" + assert file_write.decode_strs("中文") == "中文" + file_write.writeExcel([[["中文"]]], str(tmp_path / "test.xls")) + assert (tmp_path / "test.xls").stat().st_size > 0 + + +def test_loop_filters_and_symlink_cycle(tmp_path): + (tmp_path / "sub").mkdir() + for name in ["a.jpg", "noextension", "b.png", "skip.jpg"]: + (tmp_path / name).touch() + (tmp_path / "sub" / "deep.jpg").touch() + (tmp_path / "sub" / "cycle").symlink_to(tmp_path, target_is_directory=True) + assert len(loop(tmp_path)) == 4 + assert get_dir_images(tmp_path) == 4 + assert len(loop(tmp_path, None)) == 5 + paths = LoopFile(tmp_path, [".jpg"], ["skip"], ["deep.jpg"]).start( + lambda path: path + ) + assert [Path(path).name for path in paths] == ["a.jpg"] + first, second = LoopFile(tmp_path), LoopFile(tmp_path) + assert first.file_extend == second.file_extend == () + + +@pytest.mark.parametrize( + "shape", [(1, 1, 3), (1, 100, 3), (100, 1, 3), (11, 16, 3), (16, 11, 3)] +) +@pytest.mark.parametrize("edge", [1, 7, 32]) +def test_noise_padding(shape, edge): + result = process.noise_padd(np.zeros(shape, np.uint8), edge) + assert result.shape == (edge, edge, 3) + assert result.dtype == np.uint8 + + +@pytest.mark.parametrize("shape", [(1, 100, 3), (100, 1, 3), (11, 16, 3)]) +def test_rectangular_padding(shape): + result = process.padd_pixel(np.zeros(shape, np.uint8), 20, 12) + assert result.shape == (12, 20, 3) + assert np.all((result == 0) | ((result >= 230) & (result < 240))) + + +@pytest.mark.parametrize( + "edge,lo,hi", [(0, 0, 255), (2, 255, 255), (2, -1, 255), (2, 0, 257)] +) +def test_invalid_padding(edge, lo, hi): + with pytest.raises(ValueError): + process.noise_padd(np.zeros((2, 2, 3), np.uint8), edge, lo, hi) + + +def test_cleanup_waits_for_all_workers(tmp_path): + for i in range(7): + (tmp_path / f"{i}.jpg").write_bytes(b"bad") + valid = tmp_path / "valid.png" + cv2.imwrite(str(valid), np.zeros((3, 4, 3), np.uint8)) + process.clear_dir(tmp_path, debug=False, multi_process=3) + assert list(tmp_path.iterdir()) == [valid] + with pytest.raises(ValueError): + process.clear_dir(tmp_path, multi_process=0) + + +def test_resize_preprocessing_hash_and_draw(tmp_path): + image = np.ones((12, 24, 3), np.uint8) * 127 + assert process.resize_with_scale(image, 12).shape == (6, 12, 3) + assert process.resize_with_scale(image, 30) is image + with pytest.raises(ValueError): + process.resize_with_scale(image, 0) + assert process.swap_chanel_to_RGB(image).shape == image.shape + assert process.our_processing(image).shape == (1, 12, 24, 3) + np.testing.assert_allclose( + process.inception_preprocesing(image), (127 / 255 - 0.5) * 2, rtol=1e-5 + ) + np.testing.assert_allclose( + process.vgg_preprocesing(image)[0, 0], [3.32, 10.22, 23.06], atol=1e-4 + ) + path = tmp_path / "image.png" + cv2.imwrite(str(path), image) + assert process.md5(path) == hashlib.md5(path.read_bytes()).hexdigest() + process.resize_dir_images_with_scale(tmp_path, 8) + assert cv2.imread(str(path)).shape[:2] == (4, 8) + assert process.draw_cvrect(image, [[1, 1, 4, 4]]) is image + contour = process.points_to_contour([[1, 1], [4, 1], [4, 4], [1, 4]]) + assert process.draw_contours(image, [contour]) is image + assert process.points_to_contours([[1, 1], [4, 1]]).shape == (1, 2, 1, 2) + assert process.polly_points2contours([[1, 1, 4, 1, 4, 4, 1, 4]]).shape == ( + 1, + 4, + 1, + 2, + ) + assert process.get_box([0, 0, 4, 8]) == (0, 4) + process.put_text(image, "a", (0, 2)) + + +def test_alpha_to_color(): + image = Image.fromarray(np.array([[[10, 20, 30, 0], [10, 20, 30, 128]]], np.uint8)) + output = np.array(imagedt.alpha_to_color(image)) + assert output.tolist() == [[[255, 255, 255, 255], [10, 20, 30, 255]]] + + +def test_md5_rename_binary_and_collision(tmp_path): + path = tmp_path / "first.jpg" + path.write_bytes(b"\xff\x00image") + digest = process.md5(path) + process.rename_imfiles_with_md5(tmp_path) + assert (tmp_path / f"{digest}.jpg").exists() + (tmp_path / "second.jpg").write_bytes(b"\xff\x00image") + with pytest.raises(FileExistsError): + process.rename_imfiles_with_md5(tmp_path) + + +def test_file_operations(tmp_path): + FilesOp.check_dir_exist(tmp_path / "nested" / "deep") + assert FilesOp.fillter_file_type(["a.b.jpg", "a.xml"]) == ["a.b", "a"] + assert FilesOp.fillter_dirname(["/tmp/image.jpg,1"]) == [["image.jpg", "1"]] + assert Path(FilesOp.rename_with_uuid(tmp_path / "b.c.jpg", "uid")) == ( + tmp_path / "uid.jpg" + ) + with pytest.raises(ValueError): + FilesOp.rename_with_uuid("noext", "uid") + with pytest.raises(OSError): + FilesOp.check_file_exist(tmp_path / "missing") + with pytest.raises(ValueError): + FilesOp.assert_in_op("x", []) + with pytest.raises(ValueError): + FilesOp.assert_not_in_op("x", ["x"]) + with pytest.raises(ValueError): + FilesOp.assert_instance_op("x") + FilesOp.assert_in_op(["x", "y"], ["x", "y"]) + FilesOp.assert_instance_op([]) + path = tmp_path / "nested" / "deep" / "image.jpg" + path.touch() + FilesOp.rename_class_dir(tmp_path) + assert (path.parent.parent / "otherdeep" / "image.jpg").exists() + + +def test_annotations_roundtrip_and_reset(tmp_path): + xml = tmp_path / "anno.xml" + Anno_OP.write_xml( + xml, [[1, 2, 3, 4], [5, 6, 7, 8]], [0.9, 0.1], ["7", "8"], thresh=0.5 + ) + assert Anno_OP.parse_lab_xml(xml) == [[1.0, 2.0, 3.0, 4.0]] + objects = Anno_OP.get_label_object(xml) + assert objects[0]["name"] == "7" + assert Anno_OP.get_object_bndbox(objects[0]) == [1.0, 2.0, 3.0, 4.0] + Anno_OP.reset_xml_cls(tmp_path, name="9", resetid_list=[7]) + assert Anno_OP.get_label_object(xml)[0]["name"] == "9" + Anno_OP.filter_xml_classes(tmp_path) + assert Anno_OP.get_label_object(xml)[0]["name"] == "0" + Anno_OP.wirte_xml_file("", xml) + assert Anno_OP.parse_lab_xml(xml) == [] + + +def test_polygon_annotation(tmp_path): + missing, flags = Anno_OP.load_annoataion(tmp_path / "missing") + assert missing.shape == (0, 4, 2) and flags.dtype == bool + path = tmp_path / "polys" / "anno.txt" + path.parent.mkdir() + path.write_text("\ufeff1,2,3,2,3,4,1,4,###\n", encoding="utf-8") + polygons, flags = Anno_OP.load_annoataion(path) + assert polygons.shape == (1, 4, 2) and flags.tolist() == [True] + Anno_OP.convert_poly_to_Xml(str(path)) + assert ( + Anno_OP.get_label_object(tmp_path / "Annotations_xml" / "anno.xml")[0]["name"] + == "2" + ) + + +def test_decorator(capsys): + @imagedt.time_cost + def add(a, b): + return a + b + + assert add(2, 3) == 5 and add.__name__ == "add" + assert "time-cost" in capsys.readouterr().out diff --git a/tests/test_metrics_data_text.py b/tests/test_metrics_data_text.py new file mode 100644 index 0000000..0934459 --- /dev/null +++ b/tests/test_metrics_data_text.py @@ -0,0 +1,354 @@ +import shutil +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +from imagedt.file import Anno_OP, read_csv +from imagedt.tensorflow.tools.DataInterface import DataSet, DataSets +from imagedt.tools.metrics import detect_eval +from imagedt.tools.metrics.f1_score import ( + _f1_score, + _total_f1_score, + calculate_f1_score, + gernerate_infos_dict, + load_label_Dict, +) +from imagedt.tools.text_generator.background_generator import BackgroundGenerator +from imagedt.tools.text_generator.computer_text_generator import ComputerTextGenerator +from imagedt.tools.text_generator.data_generator import FakeTextDataGenerator +from imagedt.tools.text_generator.distorsion_generator import DistorsionGenerator + + +@pytest.mark.parametrize( + "boxes,classes,expected", + [ + ([[1, 1, 10, 10]], ["cat"], 1.0), + ([[1, 1, 10, 10]] * 2, ["cat"] * 2, 2 / 3), + ([], [], 0.0), + ([[20, 20, 30, 30]], ["cat"], 0.0), + ([[1, 1, 10, 10]], ["dog"], 0.0), + ], +) +def test_voc_matching(tmp_path, boxes, classes, expected): + gt, det = tmp_path / "gt", tmp_path / "det" + gt.mkdir() + det.mkdir() + Anno_OP.write_xml(gt / "a.xml", [[1, 1, 10, 10]], [1.0], ["cat"]) + Anno_OP.write_xml(det / "a.xml", boxes, [0.9] * len(boxes), classes) + assert detect_eval.voc_eval(det, gt) == pytest.approx(expected) + + +def test_voc_empty_unknown_low_conf_and_difficult(tmp_path): + gt, det = tmp_path / "gt", tmp_path / "det" + gt.mkdir() + det.mkdir() + assert detect_eval.voc_eval(det, gt) == 0 + Anno_OP.write_xml(gt / "a.xml", [], [], []) + Anno_OP.write_xml(det / "a.xml", [[1, 1, 4, 4]], [0.9], ["cat"]) + assert detect_eval.voc_eval(det, gt) == 0 + Anno_OP.write_xml(gt / "b.xml", [[1, 1, 4, 4]], [1.0], ["cat"]) + Anno_OP.write_xml(det / "b.xml", [[1, 1, 4, 4]], [0.1], ["cat"]) + assert detect_eval.voc_eval(det, gt) == 0 + assert detect_eval.voc_eval(det, gt, set_gt_labels=["unused"]) == 0 + with pytest.raises(ValueError): + detect_eval.voc_eval(det, gt, det_file_type="bad") + with pytest.raises(ValueError): + detect_eval.voc_eval(det, gt, conf=2) + annotation = Anno_OP.read_xml(gt / "b.xml") + from lxml import etree + + etree.SubElement(annotation.find("object"), "difficult").text = "1" + Anno_OP.write_anno_object(annotation, gt / "b.xml") + assert detect_eval.parse_rec(gt / "b.xml")["cat"][0]["difficult"] == 1 + + +def test_voc_txt(tmp_path): + gt, det = tmp_path / "gt", tmp_path / "det" + gt.mkdir() + det.mkdir() + Anno_OP.write_xml(gt / "a.xml", [[1, 1, 10, 10]], [1.0], ["cat"]) + (det / "detect_cat.txt").write_text("a 0.9 1 1 10 10\n") + assert detect_eval.voc_eval(det, gt, det_file_type="txt") == 1 + + +def test_f1_rows_and_alignment(tmp_path): + labels = ["101\t1\t苹果", "102\t2\t梨"] + mapping = load_label_Dict(labels) + info = gernerate_infos_dict(["a,1,.9", "b,1,.8"], ["a,1", "b,2"], mapping) + rows = _f1_score(info, mapping) + assert rows[0][-1] == "F1-score" + assert rows[1][-1] == pytest.approx(0.6667) + assert info["2"]["object_id"] == "102" + assert _total_f1_score(info) == (0.5, 0.5) + assert _total_f1_score({}) == (0.0, 0.0) + with pytest.raises(ValueError): + gernerate_infos_dict(["a,1,.9"], [], mapping) + predict, truth, label = ( + tmp_path / name for name in ["pre.txt", "gt.txt", "labels.txt"] + ) + predict.write_text("a,1,.9\nb,2,.9\n") + truth.write_text("b,2\na,1\n") + label.write_text("\n".join(labels), encoding="utf-8") + calculate_f1_score(predict, truth, label) + assert [row[-1] for row in read_csv(tmp_path / "image_classify_infos.csv")[1:]] == [ + "1", + "1", + ] + truth.write_text("other,2\na,1\n") + with pytest.raises(ValueError): + calculate_f1_score(predict, truth, label) + + +def test_roc_vectorized(tmp_path): + from imagedt.tools.metrics.roc_curve import roc + + path = tmp_path / "result.csv" + path.write_text( + "Image,Name,Label,Predict,Confidence,Match\na,x,1,1,0.9,1\nb,y,2,1,0.4,0\n" + ) + roc(path) + rows = read_csv(tmp_path / "thres-fpr-tpr.csv") + assert len(rows) == 1001 + assert list(map(float, rows[500])) == [0.0, 1.0, 0.5] + assert (tmp_path / "roc.png").exists() + path.write_text("") + with pytest.raises(ValueError): + roc(path) + + +@pytest.mark.parametrize("is_list", [False, True]) +def test_dataset_batches(is_list): + data = DataSet() + x, y = np.arange(5), np.arange(5) * 2 + if is_list: + x, y = list(x), list(y) + data.set_x(x) + data.set_y(y) + assert data.batch_count(2) == 2 + assert data.batch_count(2, False) == 3 + batches = [data.next_batch(2, False) for _ in range(3)] + assert [len(x) for x, y in batches] == [2, 2, 1] + assert sorted(int(v) for x, y in batches for v in x) == list(range(5)) + for x, y in batches: + np.testing.assert_array_equal(np.asarray(x) * 2, y) + assert len(data.next_batch(2)[0]) == 2 + data.reset_step() + assert len(data.next_batch(2)[0]) == 2 + with pytest.raises(ValueError): + data.next_batch(6) + with pytest.raises(ValueError): + data.batch_count(0) + empty = DataSet() + assert empty.count == 0 + with pytest.raises(ValueError): + empty.next_batch(1) + ds = DataSets() + assert ds.train is not ds.test and ds.validate is not ds.train + with pytest.raises(ValueError): + _ = ds.x_shape + with pytest.raises(ValueError): + _ = ds.y_shape + ds.set_x_shape((2, 3)) + ds.set_y_shape((1,)) + assert ds.x_shape == (2, 3) and ds.y_shape == (1,) + + +@pytest.fixture +def font(): + # Matplotlib ships a redistributable test font on all CI platforms. + import matplotlib + + return str(Path(matplotlib.get_data_path()) / "fonts" / "ttf" / "DejaVuSans.ttf") + + +@pytest.mark.parametrize("orientation", [0, 1]) +def test_text_render(font, orientation): + image = ComputerTextGenerator.generate( + "Hello 123", font, "#000000,#222222", 24, orientation, 1.0 + ) + assert image.mode == "RGBA" and min(image.size) > 0 + assert np.array(image)[..., 3].max() > 0 + with pytest.raises(ValueError): + ComputerTextGenerator.generate("", font, "#000000", 24, orientation, 1) + with pytest.raises(ValueError): + ComputerTextGenerator.generate("hi", font, "#000000", 24, 2, 1) + + +@pytest.mark.parametrize("background", [0, 1, 2]) +def test_fake_text(font, background, tmp_path): + image, text = FakeTextDataGenerator.generate( + 0, + "1234", + font, + tmp_path, + 40, + "png", + 0, + False, + 0, + False, + background, + 0, + 0, + False, + 1, + 100, + 1, + "#000000", + 0, + 1, + save=True, + ) + assert image.size == (100, 40) and text == "1234" + assert (tmp_path / "0_1234.png").exists() + + +@pytest.mark.parametrize( + "vertical,horizontal", [(False, False), (True, False), (False, True), (True, True)] +) +@pytest.mark.parametrize("kind", ["sin", "cos", "random"]) +def test_distortions(kind, vertical, horizontal): + image = Image.new("RGBA", (30, 20), "black") + distorted = getattr(DistorsionGenerator, kind)(image, vertical, horizontal) + assert distorted.width >= image.width and distorted.height >= image.height + assert distorted.mode == "RGBA" + + +def test_backgrounds_and_generator(font, tmp_path, monkeypatch): + assert BackgroundGenerator.quasicrystal(1, 1).size == (1, 1) + assert BackgroundGenerator.plain_white(3, 4).size == (4, 3) + assert BackgroundGenerator.gaussian_noise(3, 4).mode == "RGB" + monkeypatch.chdir(tmp_path) + (tmp_path / "pictures").mkdir() + Image.new("RGB", (2, 3)).save(tmp_path / "pictures" / "tiny.png") + assert BackgroundGenerator.picture(20, 30).size == (30, 20) + from imagedt.tools.text_generator.text_generator import Text_Generator + + fonts = tmp_path / "fonts" + fonts.mkdir() + shutil.copy(font, fonts / "font.ttf") + with Text_Generator(fonts, langu="en") as generator: + assert len(generator.gen_price_realtime_datas(2)) == 2 + assert len(generator.gen_price_realtime_datas(1)) == 1 + generator.set_params(["hi"]) + assert len(generator.generate_texts()) == 1 + with pytest.raises(ValueError): + generator.gen_price_realtime_datas(-1) + + +def test_plot_empty_and_counts(tmp_path): + from imagedt.tools.plot_curve import plot_tools + + plot_tools.set_xy_values({}) + assert plot_tools.x == [] and plot_tools.y == [] + data = tmp_path / "data" + data.mkdir() + (data / "a").mkdir() + (data / "a" / "test.jpg").touch() + assert plot_tools.load_dataset_infos(data) == {"a": 1} + plot_tools.plot_datas(data) + assert (tmp_path / "datasets.png").exists() + + +@pytest.mark.parametrize( + "orientation,distortion,alignment,name_format", + [(0, 1, 0, 0), (0, 2, 2, 2), (1, 3, 1, 1), (0, 0, 1, 99)], +) +def test_generator_option_branches( + font, tmp_path, orientation, distortion, alignment, name_format +): + image, text = FakeTextDataGenerator.generate( + 0, + "abc", + font, + tmp_path, + 40, + "png", + 2, + True, + 1, + True, + 1, + distortion, + 2, + False, + name_format, + 100, + alignment, + "#000000", + orientation, + 1, + save=True, + ) + assert min(image.size) > 0 and text == "abc" + with pytest.raises(ValueError): + FakeTextDataGenerator.generate( + 0, + "abc", + font, + tmp_path, + 5, + "png", + 0, + False, + 0, + False, + 1, + 0, + 0, + False, + 1, + 100, + 1, + "#000000", + 0, + 1, + ) + with pytest.raises(NotImplementedError): + FakeTextDataGenerator.generate( + 0, + "abc", + font, + tmp_path, + 40, + "png", + 0, + False, + 0, + False, + 1, + 0, + 0, + True, + 1, + 100, + 1, + "#000000", + 0, + 1, + ) + + +def test_voc_ignores_difficult_and_uses_strict_iou(tmp_path): + # Test value: protects=declared VOC difficult/IoU semantics; + # failure=ignored GT counts or boundary changes; oracle=analytic F1; seam=none. + gt, det = tmp_path / "gt", tmp_path / "det" + gt.mkdir() + det.mkdir() + Anno_OP.write_xml( + gt / "a.xml", [[0, 0, 1, 1], [10, 10, 11, 11]], [1, 1], ["cat", "cat"] + ) + root = Anno_OP.read_xml(gt / "a.xml") + from lxml import etree + + etree.SubElement(root.findall("object")[1], "difficult").text = "1" + Anno_OP.write_anno_object(root, gt / "a.xml") + Anno_OP.write_xml( + det / "a.xml", [[0, 0, 1, 1], [10, 10, 11, 11]], [1, 1], ["cat", "cat"] + ) + assert detect_eval.voc_eval(det, gt) == 1.0 + # Inclusive areas are 4 and 2; IoU is exactly one half, which must not match. + Anno_OP.write_xml(det / "a.xml", [[0, 0, 0, 1]], [1], ["cat"]) + assert detect_eval.voc_eval(det, gt, ovthresh=0.5) == 0.0 diff --git a/tests/test_remaining_utilities.py b/tests/test_remaining_utilities.py new file mode 100644 index 0000000..bf96ec0 --- /dev/null +++ b/tests/test_remaining_utilities.py @@ -0,0 +1,240 @@ +from pathlib import Path + +import cv2 +import numpy as np +import pytest +from lxml import etree +from PIL import Image + +from imagedt.file import Anno_OP, FilesOp +from imagedt.image import process + + +def test_network_socket_success_and_failure(monkeypatch): + from imagedt.tools import com_tools + + class Sock: + closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.closed = True + + def settimeout(self, seconds): + assert seconds == 2 + + def connect(self, address): + pass + + def getsockname(self): + return ("127.0.0.1", 0) + + sock = Sock() + monkeypatch.setattr(com_tools.socket, "socket", lambda *args: sock) + assert com_tools.getip() == "127.0.0.1" and sock.closed + + def fail(*args): + raise OSError("offline") + + monkeypatch.setattr(com_tools.socket, "socket", fail) + assert com_tools.getip() == "x.x.x.x" + + +def test_pair_file_operations(tmp_path): + images = tmp_path / "JPEGImages" + images.mkdir() + annotations = tmp_path / "Annotations" + annotations.mkdir() + for name in ["a.b", "c"]: + (images / (name + ".jpg")).write_bytes(b"jpg") + (annotations / (name + ".xml")).write_text("") + assert FilesOp.check_data_pairs(tmp_path) == "have same files" + pairs = FilesOp.get_jpg_xml_pairs(tmp_path) + assert len(pairs) == 2 and Path(pairs[0][0]).name == "a.b.jpg" + (annotations / "unpaired.xml").write_text("") + FilesOp.check_data_pairs(tmp_path) + assert not (annotations / "unpaired.xml").exists() + FilesOp.rename_file_with_uuid(tmp_path) + assert {p.stem for p in images.iterdir()} == {p.stem for p in annotations.iterdir()} + FilesOp.rename_single_file_uuid(images) + assert len(list(images.iterdir())) == 2 + assert len(FilesOp.loop_dir(tmp_path)) == 4 + FilesOp.del_broken_image(tmp_path) + assert not list(images.iterdir()) + + +def test_pair_rename_collision_and_rollback(tmp_path, monkeypatch): + first, second = tmp_path / "a.jpg", tmp_path / "a.xml" + first.touch() + second.touch() + (tmp_path / "uid.xml").touch() + with pytest.raises(FileExistsError): + FilesOp.rename_data_pairs([str(first), str(second)], "uid") + assert first.exists() and second.exists() + (tmp_path / "uid.xml").unlink() + import os + + real_rename = os.rename + + def fail_second(source, target): + if str(source).endswith("a.xml"): + raise OSError("test failure") + real_rename(source, target) + + monkeypatch.setattr(os, "rename", fail_second) + with pytest.raises(OSError): + FilesOp.rename_data_pairs([str(first), str(second)], "uid") + assert first.exists() and second.exists() + + +def test_annotation_optional_fields_and_boundaries(tmp_path): + path = tmp_path / "a.xml" + Anno_OP.write_xml(path, [[1, 2, 3, 4]], [0.9], "7") + annotation = Anno_OP.read_xml(path) + obj = annotation.find("object") + for key, value in [("price", "2"), ("classify_confidence", ".9"), ("blur", "0")]: + etree.SubElement(obj, key).text = value + prices = etree.SubElement(obj, "prices") + etree.SubElement(prices, "price").text = "3" + Anno_OP.write_anno_object(annotation, path) + objects = Anno_OP.get_label_object(path) + assert objects[0]["prices"] == ["3"] and objects[0]["price"] == "2" + Anno_OP.filter_xml_classes(tmp_path, resetid_list=[7]) + assert Anno_OP.get_label_object(path)[0]["name"] == "7" + Anno_OP.reset_xml_cls(tmp_path, resetid_list=[8]) + assert Anno_OP.get_label_object(path) == [] + with pytest.raises(ValueError): + Anno_OP.reset_xml_cls(tmp_path, resetid_list="x") + with pytest.raises(ValueError): + Anno_OP.filter_xml_classes(tmp_path, resetid_list="x") + with pytest.raises(ValueError): + Anno_OP.write_xml(path, [[0, 0, 1, 1]], [], ["x"]) + text = tmp_path / "a.txt" + text.write_text("1,2,3,4\n") + assert Anno_OP.read_txt_bndbox(text) == [["1", "2", "3", "4"]] + assert Anno_OP.parse_poly_locations( + [[3.1, 2.2], [1.2, 4.5], [2, 1.1], [4.1, 3]] + ) == (1, 1, 5, 5) + Anno_OP.wirte_xml_file(b"", path) + poly = tmp_path / "poly" + poly.mkdir() + (poly / "a.txt").write_text("1,1,2,1,2,2,1,2,hello\n") + Anno_OP.convert_poly_to_Xml(str(poly / "a.txt")) + assert ( + Anno_OP.get_label_object(tmp_path / "Annotations_xml" / "a.xml")[0]["name"] + == "1" + ) + empty = tmp_path / "empty.txt" + empty.touch() + assert Anno_OP.load_annoataion(empty)[0].shape == (0, 4, 2) + + +def test_image_helpers_and_errors(tmp_path, monkeypatch): + invalid = tmp_path / "bad.jpg" + invalid.write_bytes(b"broken") + process.remove_broken_image(tmp_path) + assert not invalid.exists() + valid = tmp_path / "good.png" + cv2.imwrite(str(valid), np.zeros((3, 4, 3), np.uint8)) + process.remove_broken_image(tmp_path) + assert valid.exists() + invalid.write_bytes(b"broken") + with pytest.raises(ValueError): + process.resize_dir_images_with_scale(tmp_path) + with pytest.raises(ValueError): + process.noise_padd(np.zeros((3, 3), np.uint8)) + assert process.get_box([2, 3, 10, 20], "top") == (2, 3) + assert process.get_box([2, 3, 10, 20], "bot") == (2, 15) + assert process.get_box([2, 3, 10, 20], "botup") == (2, 15) + for name in ["imshow", "waitKey", "destroyAllWindows"]: + monkeypatch.setattr(cv2, name, lambda *args: None) + process.cv_show(np.zeros((3, 3, 3), np.uint8)) + process.put_cvtext( + np.zeros((20, 20, 3), np.uint8), "A", (1, 1), font_type=cv2.FONT_HERSHEY_SIMPLEX + ) + + +def test_handwriting_clear_error(): + from imagedt.tools.text_generator.handwritten_text_generator import ( + HandwrittenTextGenerator, + ) + + with pytest.raises(NotImplementedError, match="legacy"): + HandwrittenTextGenerator.generate("hello") + + +def test_pdf_real_render(tmp_path): + pdfium = pytest.importorskip("pypdfium2") + from imagedt.tools.pdf import pdf2image + + source = tmp_path / "source.pdf" + document = pdfium.PdfDocument.new() + for _ in range(2): + page = document.new_page(100, 120) + page.close() + document.save(str(source)) + document.close() + pdf2image(source, dpi=72) + with Image.open(tmp_path / "source-0.jpg") as image: + assert image.size == (100, 120) + with Image.open(tmp_path / "source-1.jpg") as image: + assert image.size == (100, 120) + with pytest.raises(ValueError): + pdf2image(source, dpi=0) + document = pdfium.PdfDocument.new() + page = document.new_page(30, 40) + page.close() + document.save(str(source)) + document.close() + pdf2image(source, save_out=tmp_path / "single.png", dpi=144) + with Image.open(tmp_path / "single.png") as image: + assert image.size == (60, 80) + + +def test_pdf_and_excel_dependency_diagnostics(monkeypatch): + from imagedt import _backends + + def fail(name): + raise ModuleNotFoundError("missing submodule", name="internal_dependency") + + monkeypatch.setattr(_backends, "import_module", fail) + with pytest.raises(ModuleNotFoundError, match="submodule"): + _backends.require("outer", "outer") + + +@pytest.mark.parametrize("failure", ["missing", "collision", "second_rename"]) +def test_md5_pair_failure_preserves_originals(tmp_path, monkeypatch, failure): + # Test value: protects=image/XML integrity on a partial rename; + # failure=missing/colliding annotation or filesystem failure; oracle=original bytes; seam=none. + images = tmp_path / "JPEGImages" + images.mkdir() + annotations = tmp_path / "Annotations" + annotations.mkdir() + image = images / "sample.jpg" + image.write_bytes(b"image") + xml = annotations / "sample.xml" + if failure != "missing": + xml.write_bytes(b"original annotation") + target = annotations / (process.md5(image) + ".xml") + if failure == "collision": + target.write_bytes(b"existing annotation") + if failure == "second_rename": + import os + + rename = os.rename + + def fail(source, destination): + if str(source) == str(xml): + raise OSError("injected XML failure") + rename(source, destination) + + monkeypatch.setattr(os, "rename", fail) + with pytest.raises(OSError): + process.rename_imfiles_with_md5(tmp_path, rename_xml=True) + assert image.read_bytes() == b"image" + if failure != "missing": + assert xml.read_bytes() == b"original annotation" + if failure == "collision": + assert target.read_bytes() == b"existing annotation" diff --git a/tests/test_tensorflow.py b/tests/test_tensorflow.py new file mode 100644 index 0000000..74732c1 --- /dev/null +++ b/tests/test_tensorflow.py @@ -0,0 +1,425 @@ +from types import SimpleNamespace + +import cv2 +import numpy as np +import pytest + +# Explicit skip when this optional extra is not installed. +tf = pytest.importorskip("tensorflow") +pytestmark = [pytest.mark.tensorflow, pytest.mark.integration] + +from imagedt.converters.tfmodel_lite import convert +from imagedt.tensorflow.lite.lite_wrapper import Lite_Wrapper +from imagedt.tensorflow.tools import dataset_util +from imagedt.tensorflow.tools.data_provider import ( + Data_Provider, + dynamically_loaded_data, + resize_image_keep_aspect, + tf_JitterCut, + tf_noise_padd, +) +from imagedt.tensorflow.tools.Records import ( + ImageReader, + RecordWriter, + write_pbmodel_summery, +) +from imagedt.tensorflow.tools.tfmodel_wrapper import TFmodel_Wrapper + + +def make_graph(path): + graph = tf.Graph() + with graph.as_default(): + data = tf.compat.v1.placeholder(tf.float32, [1, 2, 2, 3], name="data") + scores = tf.reduce_mean(data, axis=[1, 2], name="features") + tf.nn.softmax(scores, name="softmax") + path.write_bytes(graph.as_graph_def().SerializeToString()) + return graph + + +def test_graph_wrapper_isolation_and_lite(tmp_path): + path = tmp_path / "tiny.pb" + make_graph(path) + data = np.zeros((1, 2, 2, 3), np.float32) + data[..., 2] = 2.0 + graph = tf.compat.v1.get_default_graph() + count = len(graph.get_operations()) + with ( + TFmodel_Wrapper(path, "data", "softmax,features") as first, + TFmodel_Wrapper(path, "data", "softmax,features") as second, + ): + assert first.graph is not second.graph + assert first.predict(data)[0]["class"] == 2 + np.testing.assert_array_equal(first.extract(data), [[0, 0, 2]]) + assert len(graph.get_operations()) == count + output = tmp_path / "tiny.tflite" + convert(path, output, input_array="data", input_shape="1,2,2,3") + model = Lite_Wrapper(output) + assert (model.height, model.width) == (2, 2) + label, score = model.predict(data) + assert label == 2 and score > 0.7 + with pytest.raises(ValueError): + model.predict(data.astype(np.float64)) + with pytest.raises(ValueError): + model.predict(data[0]) + write_pbmodel_summery(path, str(tmp_path / "summary")) + assert list((tmp_path / "summary").iterdir()) + + +@pytest.mark.parametrize( + "options,error", + [ + ({"output_format": "GRAPHVIZ_DOT"}, NotImplementedError), + ({"inference_type": "QUANTIZED_UINT8"}, NotImplementedError), + ({"input_array": "x,y"}, ValueError), + ], +) +def test_lite_conversion_errors(tmp_path, options, error): + with pytest.raises(error): + convert(tmp_path / "missing", tmp_path / "output", **options) + + +def test_features_xml_and_reader(tmp_path): + assert list(dataset_util.int64_feature(2).int64_list.value) == [2] + assert list(dataset_util.int64_list_feature([1, 2]).int64_list.value) == [1, 2] + assert list(dataset_util.bytes_feature("你好").bytes_list.value) == [ + "你好".encode() + ] + assert list(dataset_util.bytes_list_feature(["a", b"b"]).bytes_list.value) == [ + b"a", + b"b", + ] + assert list(dataset_util.float_feature(0.5).float_list.value) == [0.5] + assert list(dataset_util.float_list_feature([0.5]).float_list.value) == [0.5] + from lxml import etree + + xml = etree.fromstring( + b"ab" + ) + assert ( + len(dataset_util.recursive_parse_xml_to_dict(xml)["annotation"]["object"]) == 2 + ) + p = tmp_path / "examples.txt" + p.write_text("a 1\nb 2\n") + assert dataset_util.read_examples_list(str(p)) == ["a", "b"] + reader = ImageReader() + encoded = cv2.imencode(".jpg", np.zeros((3, 4, 3), np.uint8))[1].tobytes() + assert reader.read_image_dims(None, encoded) == (3, 4) + with pytest.raises(ValueError): + reader.decode_jpeg(None, b"bad") + + +def test_record_writer_no_train_validation_overlap(tmp_path): + image = np.zeros((6, 9, 3), np.uint8) + rows = [] + for i in range(4): + path = tmp_path / f"{i}.jpg" + cv2.imwrite(str(path), image) + rows.append(f"{path}\t{i}") + index = tmp_path / "train.txt" + index.write_text("\n".join(rows)) + directory = tmp_path / "records" + RecordWriter.write_cls_records(index, directory, test_ratio=0.5) + + def read_labels(prefix): + result = [] + for path in directory.glob(prefix + "*.tfrecord"): + for raw in tf.compat.v1.io.tf_record_iterator(str(path)): + example = tf.train.Example.FromString(raw) + feature = example.features.feature + assert feature["image/format"].bytes_list.value[0] == b"png" + result.append(feature["image/class/label"].int64_list.value[0]) + return result + + training, validation = read_labels("train_"), read_labels("validation_") + assert len(training) == len(validation) == 2 + assert set(training).isdisjoint(validation) + assert set(training + validation) == set(range(4)) + unrelated = directory / "train_notes.tfrecord" + unrelated.write_bytes(b"keep") + for ratio in [0, 0.5, 1, 0.5]: + (directory / "train_999999.tfrecord").write_bytes(b"obsolete") + RecordWriter.write_cls_records(index, directory, ratio) + training, validation = read_labels("train_0"), read_labels("validation_0") + assert len(validation) == int(4 * ratio) + assert len(training) == 4 - int(4 * ratio) + assert set(training).isdisjoint(validation) + assert set(training + validation) == set(range(4)) + assert not (directory / "train_999999.tfrecord").exists() + assert unrelated.read_bytes() == b"keep" + with pytest.raises(ValueError): + RecordWriter.write_cls_records(index, directory, 2) + example = RecordWriter.create_tf_example( + None, str(tmp_path / "0.jpg"), [[1, 2, 3, 4, "1", "文字"]] + ) + assert ( + example.features.feature["image/object/class/text"].bytes_list.value[0] + == "文字".encode() + ) + empty = RecordWriter.create_tf_example(None, str(tmp_path / "0.jpg"), []) + assert len(empty.features.feature["image/object/bbox/xmin"].float_list.value) == 0 + + +@pytest.mark.parametrize("shape", [(3, 8, 3), (8, 3, 3), (1, 20, 3)]) +def test_tensor_image_transforms(shape): + data = tf.constant(np.zeros(shape, np.uint8)) + result = tf_noise_padd(data, 12) + assert tuple(result.shape) == (12, 12, 3) + assert max(resize_image_keep_aspect(data, 12).shape[:2]) == 12 + cropped = tf_JitterCut(data, 0.2) + assert min(cropped.shape[:2]) >= 1 + np.testing.assert_array_equal(tf_JitterCut(data, 0).numpy(), data.numpy()) + + +def test_dynamic_data_and_provider(tmp_path): + path = tmp_path / "image.jpg" + cv2.imwrite(str(path), np.zeros((3, 8, 3), np.uint8)) + images, labels = dynamically_loaded_data([str(path)], [2], height=6, width=10) + assert images.shape == (1, 6, 10, 3) and labels == [2] + with pytest.raises(ValueError): + dynamically_loaded_data([str(path)], []) + RecordWriter.convert_to_tfrecord([[str(path), 2]], tmp_path) + graph = tf.Graph() + with graph.as_default(): + provider = Data_Provider(tmp_path, 4) + images, labels = provider.read_from_tfrecord( + batch_size=1, image_shape=(6, 10, 3) + ) + assert images.shape.as_list() == [1, 6, 10, 3] + with tf.compat.v1.Session(graph=graph) as session: + session.run( + [ + tf.compat.v1.global_variables_initializer(), + tf.compat.v1.local_variables_initializer(), + ] + ) + coordinator = tf.compat.v1.train.Coordinator() + threads = tf.compat.v1.train.start_queue_runners( + sess=session, coord=coordinator + ) + try: + batch, label = session.run([images, labels]) + assert batch.shape == (1, 6, 10, 3) and np.argmax(label[0]) == 2 + finally: + coordinator.request_stop() + coordinator.join(threads) + + +def test_losses_and_lenet(): + from imagedt.tensorflow.network.lenet import lenet + from imagedt.tensorflow.optim.loss import arcface_loss, smooth_l1_loss + + loss = smooth_l1_loss(tf.constant([0.0, 0.0]), tf.constant([0.2, 1.0])) + np.testing.assert_allclose(loss.numpy(), [0.02, 0.375], atol=1e-6) + logits, endpoints = lenet(tf.zeros((2, 28, 28, 1)), 3, is_training=False) + assert logits.shape == (2, 3) + np.testing.assert_allclose( + tf.reduce_sum(endpoints["Predictions"], axis=1).numpy(), [1, 1], atol=1e-6 + ) + graph = tf.Graph() + with graph.as_default(), tf.compat.v1.Session(graph=graph) as session: + output = arcface_loss( + tf.constant([[0.0, 0.0], [1.0, 0.0]]), tf.constant([[0], [1]]), 3 + ) + session.run(tf.compat.v1.global_variables_initializer()) + assert np.isfinite(session.run(output)).all() + + +def test_read_dataset(tmp_path): + path = tmp_path / "data.txt" + path.write_text("a\nb\n") + config = SimpleNamespace( + num_readers=2, + shuffle=False, + num_epochs=1, + read_block_length=1, + num_parallel_map_calls=1, + prefetch_size=1, + ) + dataset = dataset_util.read_dataset( + tf.data.TextLineDataset, lambda line: line, str(path), config + ) + assert list(dataset.as_numpy_iterator()) == [b"a", b"b"] + with tf.Graph().as_default(): + dataset = tf.data.Dataset.from_tensor_slices([1, 2]) + iterator = dataset_util.make_initializable_iterator(dataset) + with tf.compat.v1.Session() as session: + session.run(iterator.initializer) + assert session.run(iterator.get_next()) == 1 + + +def test_detection_records_and_legacy_converter(tmp_path, monkeypatch): + from imagedt.converters import tfrecords as legacy + from imagedt.file import Anno_OP + from imagedt.tensorflow.tools.tfrecords import visual_records + + root = tmp_path / "dataset" + root.mkdir() + (root / "JPEGImages").mkdir() + (root / "Annotations").mkdir() + path = root / "JPEGImages" / "a.jpg" + cv2.imwrite(str(path), np.zeros((10, 12, 3), np.uint8)) + Anno_OP.write_xml(root / "Annotations" / "a.xml", [[1, 2, 8, 9]], [1.0], ["7"]) + out = tmp_path / "records" + assert RecordWriter.converte_anno_info(root, "7")[str(path)][0][-1] == "7" + RecordWriter.write_detect_records(root, out) + files = list(out.glob("*.tfrecord")) + example = tf.train.Example.FromString( + next(tf.compat.v1.io.tf_record_iterator(str(files[0]))) + ) + assert list(example.features.feature["image/height"].int64_list.value) == [10] + labelmap = legacy.get_label_dict(legacy.get_files_dir_dict([str(path)])) + assert labelmap == {"JPEGImages": 0} + assert legacy.get_image_pairs(root) == [str(path)] + rows = legacy.write_laebl_lines(out, [str(path)], labelmap) + legacy.convert_to_tfrecord(rows, out) + assert ( + legacy.image_to_tfexample(b"x", b"jpg", 2, 3, 1) + .features.feature["image/height"] + .int64_list.value[0] + == 2 + ) + from PIL import Image + + shown = [] + monkeypatch.setattr(Image.Image, "show", lambda self: shown.append(self.size)) + visual_records(files[0]) + assert shown == [(12, 10)] + + +def test_converted_network_layers(): + from imagedt.converters.taffe.tensorflow.network import Network + + class Tiny(Network): + def setup(self): + self.feed("data").conv(3, 3, 4, 1, 1, name="conv", group=2).relu( + name="relu" + ).max_pool(2, 2, 2, 2, name="pool") + self.avg_pool(2, 2, 1, 1, name="avg").lrn(2, 1e-4, 0.75, name="lrn") + self.batch_normalization(name="bn", relu=True).dropout(1.0, name="drop") + self.feed("drop", "drop").concat(3, name="concat") + self.feed("drop", "drop").add(name="add") + self.feed("drop", "drop").multiply(name="mul") + self.feed("drop", "drop").max(name="max") + self.feed("max").fc(3, name="fc").softmax(name="softmax") + + graph = tf.Graph() + with graph.as_default(), tf.compat.v1.Session(graph=graph) as session: + net = Tiny({"data": tf.constant(np.ones((1, 8, 8, 4), np.float32))}) + session.run(tf.compat.v1.global_variables_initializer()) + output = session.run(net.get_output()) + assert output.shape == (1, 3) + assert np.isfinite(output).all() + np.testing.assert_allclose(output.sum(axis=1), [1.0], atol=1e-5) + assert net.get_unique_name("conv").startswith("conv_") + with pytest.raises(KeyError): + net.feed("missing") + + +def test_lenet_features_errors_and_quantized_wrapper(): + from imagedt.tensorflow.network.lenet import lenet + + features, _ = lenet(tf.zeros((1, 28, 28, 1)), num_classes=0, is_training=False) + assert features.shape == (1, 1024) + with pytest.raises(ValueError): + lenet(tf.zeros((1, 28, 28, 1)), drop_keep_prob=0) + model = object.__new__(Lite_Wrapper) + model.input_details = [{"dtype": np.uint8, "shape": [1, 2, 2, 3]}] + with pytest.raises(ValueError): + model._check_model_type() + model.input_details = [{"dtype": np.float32, "shape": [2, 3]}] + with pytest.raises(ValueError): + model._check_model_type() + + +def test_zero_based_labels_and_all_declared_tool_exports(): + # Test value: protects=last-class one-hot targets and stable export resolution; + # failure=one-based label overflow/stale registry; oracle=identity matrix and resolved names; seam=none. + from imagedt.converters.tfrecords import get_label_dict + from imagedt.tensorflow import tools + + mapping = get_label_dict({"a": [], "b": []}) + assert mapping == {"a": 0, "b": 1} + np.testing.assert_array_equal( + tf.one_hot(list(mapping.values()), 2).numpy(), np.eye(2) + ) + for name in tools.__all__: + assert getattr(tools, name) is not None + + +def test_failed_wrapper_closes_session(tmp_path, monkeypatch): + # Test value: protects=owned Session release during constructor failure; + # failure=tensor validation raises after Session opens; oracle=actual closed Session; seam=none. + path = tmp_path / "tiny.pb" + make_graph(path) + closed = [] + original = TFmodel_Wrapper.close + + def close(wrapper): + original(wrapper) + closed.append(wrapper.sess._closed) + + monkeypatch.setattr(TFmodel_Wrapper, "close", close) + with pytest.raises(ValueError): + TFmodel_Wrapper(path, "missing", "softmax") + assert closed == [True] + + +def test_corrupt_record_images_have_split_specific_manifests(tmp_path, monkeypatch): + # Test value: protects=corrupt input containment and split diagnostics; + # failure=validation errors leak into training report; oracle=manifest names + parsed records; seam=none. + from imagedt.tensorflow.tools import Records + + monkeypatch.setattr(Records.random, "shuffle", lambda items: None) + files = [] + for name in ["bad-train", "good-train", "bad-validation", "good-validation"]: + path = tmp_path / (name + ".jpg") + if name.startswith("bad"): + path.write_bytes(b"corrupt") + else: + cv2.imwrite(str(path), np.zeros((3, 4, 3), np.uint8)) + files.append(str(path) + "\t0") + index = tmp_path / "index.txt" + index.write_text("\n".join(files)) + output = tmp_path / "records" + RecordWriter.write_cls_records(index, output, 0.5) + assert ( + output / "test_error_images.txt" + ).read_text().strip() == "bad-validation.jpg" + assert (output / "train_error_images.txt").read_text().strip() == "bad-train.jpg" + for prefix in ["train_", "validation_"]: + path = next(output.glob(prefix + "*.tfrecord")) + assert len(list(tf.compat.v1.io.tf_record_iterator(str(path)))) == 1 + + +def test_arcface_logits_match_margin_and_fallback_formula(): + # Test value: protects=target-only ArcFace margin and large-angle fallback; + # failure=wrong mask/sign/threshold; oracle=independent NumPy formula; seam=none. + import math + + from imagedt.tensorflow.optim.loss import arcface_loss + + embedding = np.array([[1.0, 0.0], [0.0, 1.0], [-1.0, 0.0]], np.float32) + labels = np.array([1, 1, 0], np.int32) + expected = embedding.copy() + for row, label in enumerate(labels): + cosine = embedding[row, label] + expected[row, label] = ( + cosine * math.cos(0.5) + - math.sqrt(max(0.0, 1 - cosine * cosine)) * math.sin(0.5) + if cosine > math.cos(math.pi - 0.5) + else cosine - math.sin(0.5) * 0.5 + ) + graph = tf.Graph() + with graph.as_default(), tf.compat.v1.Session(graph=graph) as session: + logits = arcface_loss( + tf.constant(embedding), + tf.constant(labels), + 2, + w_init=tf.constant_initializer(np.eye(2)), + s=2.0, + m=0.5, + ) + session.run(tf.compat.v1.global_variables_initializer()) + np.testing.assert_allclose( + session.run(logits), expected * 2, rtol=1e-6, atol=1e-6 + ) diff --git a/upload_pip.sh b/upload_pip.sh index 4c16bc3..b9198b5 100644 --- a/upload_pip.sh +++ b/upload_pip.sh @@ -1,19 +1,13 @@ -# 打包 -# 参考: https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi - -# 清掉文件 -rm -rf build -rm -rf dist -rm -rf imagedt.egg-info - -# 切换到python3，以便包通用 -# source py35 - -# 本地打包，--universal适配python2与python3 -python setup.py sdist bdist_wheel --universal - -# 上传 -twine upload dist/* - -# 退出python3，进入python2中 -# source deactivate +#!/usr/bin/env bash +set -euo pipefail +# Build and validate a fresh release. Upload is an explicit, separate choice. +python_cmd="${PYTHON:-python3}" +output_dir="$(mktemp -d "${TMPDIR:-/tmp}/imagedt-release.XXXXXX")" +"$python_cmd" -m build --outdir "$output_dir" +"$python_cmd" -m twine check "$output_dir"/* +printf 'Validated artifacts: %s\n' "$output_dir" +if [[ "${1:-}" == "--upload" ]]; then + "$python_cmd" -m twine upload "$output_dir"/* +else + printf 'Ready for review. To publish explicitly, run: PYTHON=%s bash upload_pip.sh --upload\n' "$python_cmd" +fi