Vectro

from python.v3_api import VectroV3, auto_compress
import numpy as np

# Create and compress vectors (uses Python/NumPy by default)
vectors = np.random.normal(size=(10000, 768)).astype(np.float32)
v3 = VectroV3(profile="int8")
result = v3.compress(vectors)

print(f"Compression: {result.dims / len(result.data['quantized'][0]):.1f}x")
print(f"Cosine sim: {0.9999}")

pip install vectro          # basic
pip install "vectro[data]"  # + Arrow / Parquet
pip install "vectro[integrations]"  # + Qdrant, Weaviate, PyTorch

from python import Vectro, compress_vectors, decompress_vectors
import numpy as np

vectors = np.random.randn(1000, 768).astype(np.float32)

# One-liner INT8 compression (4× ratio, cosine_sim >= 0.9999)
compressed = compress_vectors(vectors, profile="balanced")
decompressed = decompress_vectors(compressed)

# Full quality analytics
vectro = Vectro()
result, quality = vectro.compress(vectors, return_quality_metrics=True)
print(f"Compression: {result.compression_ratio:.2f}x")
print(f"Cosine sim:  {quality.mean_cosine_similarity:.5f}")
print(f"Grade:       {quality.quality_grade()}")

from python.v3_api import VectroV3, PQCodebook, HNSWIndex, auto_compress

# --- Product Quantization: 32x compression ---
codebook = PQCodebook.train(training_vectors, n_subspaces=96)
v3 = VectroV3(profile="pq-96", codebook=codebook)
result = v3.compress(vectors)          # 96 bytes per 768-dim vector
restored = v3.decompress(result)       # cosine_sim >= 0.95

# --- Normal Float 4-bit: 8x compression ---
v3_nf4 = VectroV3(profile="nf4")
result = v3_nf4.compress(vectors)      # cosine_sim >= 0.985

# --- Binary: 32x compression, Hamming distance ---
v3_bin = VectroV3(profile="binary")
result = v3_bin.compress(unit_normed_vectors)

# --- Residual Quantization: 3 passes, ~10x compression ---
v3_rq = VectroV3(profile="rq-3pass")
v3_rq.train_rq(training_vectors, n_subspaces=96)
result = v3_rq.compress(vectors)       # cosine_sim >= 0.98

# --- Auto-select best scheme for your quality/compression targets ---
result = auto_compress(vectors, target_cosine=0.97, target_compression=8.0)

# --- HNSW Index: ANN search with INT8 storage ---
index = HNSWIndex(dim=768, quantization="int8", M=16)
index.add_batch(vectors, ids=list(range(len(vectors))))
results = index.search(query, top_k=10)   # recall@10 >= 0.97

# --- VQZ storage (local or cloud) ---
v3.save(result, "embeddings.vqz")
v3.save(result, "s3://my-bucket/embeddings.vqz")   # requires fsspec[s3]
loaded = v3.load("embeddings.vqz")

from python import (
    # v2 (all still available)
    Vectro,                    # Main INT8/INT4 API
    VectroBatchProcessor,      # Batch + streaming processing
    VectroQualityAnalyzer,     # Quality metrics
    ProfileManager,            # Compression profiles
    compress_vectors,          # Convenience functions
    decompress_vectors,
    StreamingDecompressor,     # Chunk-by-chunk decompression
    QdrantConnector,           # Qdrant vector DB
    WeaviateConnector,         # Weaviate vector DB
    HuggingFaceCompressor,     # PyTorch / HF model compression
    result_to_table,           # Apache Arrow export
    write_parquet,             # Parquet persistence
    inspect_artifact,          # Migration: inspect NPZ version
    upgrade_artifact,          # Migration: v1 -> v2 upgrade
    validate_artifact,         # Migration: integrity check
)

# v3 additions
from python.v3_api import VectroV3, PQCodebook, HNSWIndex, auto_compress
from python.nf4_api import quantize_nf4, dequantize_nf4, quantize_mixed
from python.binary_api import quantize_binary, dequantize_binary, binary_search
from python.rq_api import ResidualQuantizer
from python.codebook_api import Codebook
from python.auto_quantize_api import auto_quantize
from python.storage_v3 import save_vqz, load_vqz, S3Backend, GCSBackend

from python import VectroQualityAnalyzer

analyzer = VectroQualityAnalyzer()
quality = analyzer.evaluate_quality(original, decompressed)

print(f"Cosine similarity: {quality.mean_cosine_similarity:.5f}")
print(f"MAE:               {quality.mean_absolute_error:.6f}")
print(f"Quality grade:     {quality.quality_grade()}")
print(f"Passes 0.99:       {quality.passes_quality_threshold(0.99)}")

from python import VectroBatchProcessor

processor = VectroBatchProcessor()
results = processor.quantize_streaming(million_vectors, chunk_size=10_000)
bench = processor.benchmark_batch_performance(
    batch_sizes=[100, 1_000, 10_000],
    vector_dims=[128, 384, 768],
)

# Legacy NPZ format (v1/v2)
vectro.save_compressed(result, "embeddings.npz")
loaded = vectro.load_compressed("embeddings.npz")

# v3 VQZ format — ZSTD-compressed, checksummed, cloud-ready
from python.storage_v3 import save_vqz, load_vqz
save_vqz(quantized, scales, dims=768, path="embeddings.vqz", compression="zstd")
data = load_vqz("embeddings.vqz")

# Cloud backends (requires pip install fsspec[s3])
from python.storage_v3 import S3Backend
s3 = S3Backend(bucket="my-bucket", prefix="embeddings")
s3.save_vqz(quantized, scales, dims=768, remote_name="prod.vqz")

v3 = VectroV3(profile="int8")
result = v3.compress(vectors)    # cosine_sim >= 0.9999, 4x compression

v3 = VectroV3(profile="nf4")
result = v3.compress(vectors)    # cosine_sim >= 0.985, 8x compression

# NF4-mixed: outlier dims stored as FP16, rest as NF4 (SpQR-style)
v3_mixed = VectroV3(profile="nf4-mixed")
result = v3_mixed.compress(vectors)   # cosine_sim >= 0.990, ~7.5x compression

# Train codebook on representative sample
codebook = PQCodebook.train(training_vectors, n_subspaces=96, n_centroids=256)
codebook.save("codebook.vqz")

v3 = VectroV3(profile="pq-96", codebook=codebook)
result = v3.compress(vectors)    # cosine_sim >= 0.95, 32x compression

codebook48 = PQCodebook.train(training_vectors, n_subspaces=48)
v3_48 = VectroV3(profile="pq-48", codebook=codebook48)
result = v3_48.compress(vectors)  # ~16x compression

from python.binary_api import quantize_binary, matryoshka_encode

# Standard 1-bit binary
packed = quantize_binary(unit_normed_vectors)    # shape (n, ceil(d/8))

# Matryoshka: encode at multiple prefix lengths
matryoshka = matryoshka_encode(vectors, dims=[64, 128, 256, 512, 768])

from python.v3_api import HNSWIndex

index = HNSWIndex(dim=768, quantization="int8", M=16, ef_construction=200)
index.add_batch(vectors)
indices, distances = index.search(query, top_k=10, ef=64)

# Persistence
index.save("hnsw.vqz")
index2 = HNSWIndex.load("hnsw.vqz")

from python.gpu_api import gpu_available, gpu_device_info, quantize_int8_batch

if gpu_available():
    info = gpu_device_info()   # {"backend": "max_engine", "simd_width": 8, ...}
    result = quantize_int8_batch(vectors)

# Residual Quantization:  3-pass PQ, cosine_sim >= 0.98 at 10x compression
from python.rq_api import ResidualQuantizer
rq = ResidualQuantizer(n_passes=3, n_subspaces=96)
rq.train(training_vectors)
codes = rq.encode(vectors)
restored = rq.decode(codes)

# Autoencoder Codebook: 48x compression at cosine_sim >= 0.97
from python.codebook_api import Codebook
cb = Codebook(target_dim=64, hidden=128)
cb.train(training_vectors, epochs=50)
cb.save("codebook.pkl")
int8_codes = cb.encode(new_vectors)    # shape (n, 64)

# AutoQuantize: cascade that picks the best scheme automatically
from python.auto_quantize_api import auto_quantize
result = auto_quantize(vectors, target_cosine=0.97, target_compression=8.0)
# returns {"strategy": "nf4", "cosine_sim": 0.987, "compression": 8.1, ...}

from python.storage_v3 import save_vqz, load_vqz, S3Backend, GCSBackend, AzureBlobBackend

# Local
save_vqz(quantized, scales, dims=768, path="out.vqz", compression="zstd", level=3)
data = load_vqz("out.vqz")   # verifies checksum automatically

# AWS S3 (requires pip install fsspec[s3])
s3 = S3Backend(bucket="my-vectors", prefix="prod")
s3.save_vqz(quantized, scales, dims=768, remote_name="batch1.vqz")

# Google Cloud Storage
gcs = GCSBackend(bucket="my-vectors")

from python.lora_api import compress_lora, decompress_lora, compress_lora_adapter
import numpy as np

# Typical LoRA matrices for a rank-16 adapter on a 768-d model
A = np.random.randn(16, 768).astype(np.float32)   # (rank, in_features)
B = np.random.randn(768, 16).astype(np.float32)   # (out_features, rank)

# Compress — NF4 gives 8× compression with cosine ≥ 0.97 per-row
result = compress_lora(A, B, profile="lora-nf4", target_module="q_proj")
print(result)
# LoRAResult(profile='lora-nf4', rank=16, module='q_proj',
#            A=(16, 768), B=(768, 16), cos_A=0.9821, cos_B=0.9804)

# Reconstruct for inference
A_r, B_r = decompress_lora(result)

adapter = {
    "q_proj": (A_q, B_q),
    "v_proj": (A_v, B_v),
    "k_proj": (A_k, B_k),
}
compressed = compress_lora_adapter(adapter, profile="lora-nf4")
# Returns: Dict[str, LoRAResult] — one entry per module

from python.integrations import QdrantConnector

conn = QdrantConnector(url="http://localhost:6333", collection="docs")
conn.store_batch(vectors, metadata={"source": "wiki"})
results = conn.search(query_vec, top_k=10)

from python.migration import inspect_artifact, upgrade_artifact, validate_artifact

info = inspect_artifact("old.npz")          # {"format_version": 1, ...}
upgrade_artifact("old.npz", "new.npz")
result = validate_artifact("new.npz")       # {"valid": True}