{"passport":{"unfragile":{"@version":"1.0","version":"2026-05","artifact":{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23","slug":"mlfoundations--mint-1t-pdf-cc-2023-23","name":"MINT-1T-PDF-CC-2023-23","type":"dataset","url":"https://huggingface.co/datasets/mlfoundations/MINT-1T-PDF-CC-2023-23","page_url":"https://unfragile.ai/mlfoundations--mint-1t-pdf-cc-2023-23","categories":["model-training"],"tags":["task_categories:image-to-text","task_categories:text-generation","language:en","license:cc-by-4.0","size_categories:1M<n<10M","format:webdataset","modality:image","modality:text","library:datasets","library:webdataset","library:mlcroissant","arxiv:2406.11271","region:us","multimodal"],"pricing":{"model":"open_source","free":true,"starting_price":null},"status":"active","verified":false},"capabilities":[{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_0","uri":"capability://data.processing.analysis.multimodal.image.text.pair.extraction.from.pdf.documents.at.scale","name":"multimodal image-text pair extraction from pdf documents at scale","description":"Extracts aligned image-text pairs from 1T+ tokens of PDF documents using a structured pipeline that preserves document layout and semantic relationships. The dataset uses WebDataset format for efficient streaming access to 633K+ samples, enabling distributed training without requiring full dataset materialization in memory. Implements MLCroissant metadata standards for reproducible dataset discovery and versioning.","intents":["Train vision-language models on real-world PDF document data with preserved layout context","Build multimodal retrieval systems that understand document structure and image-text relationships","Create datasets for document understanding tasks like table extraction, figure captioning, and layout analysis","Benchmark image-to-text and text-generation models on document-scale multimodal data"],"best_for":["ML researchers training vision-language foundation models","Teams building document AI and OCR systems","Organizations developing multimodal RAG systems for enterprise document processing","Researchers studying document layout understanding and spatial reasoning"],"limitations":["633K samples represent filtered subset of Common Crawl — not exhaustive coverage of all document types or languages","PDF extraction quality depends on source document structure; scanned/image-based PDFs may have degraded text alignment","WebDataset streaming format requires compatible data loading libraries; not directly compatible with standard PyTorch DataLoader without adapters","CC-BY-4.0 license requires attribution; commercial use requires compliance with original source licenses","No built-in deduplication or quality filtering — downstream users must implement their own data cleaning pipelines"],"requires":["Python 3.8+","HuggingFace datasets library (>=2.14.0)","WebDataset library (>=0.2.0) for efficient streaming","~500GB+ disk space for full dataset download or streaming access via HuggingFace Hub","MLCroissant-compatible metadata reader for reproducible dataset versioning"],"input_types":["PDF documents from Common Crawl 2023","Structured metadata in MLCroissant format"],"output_types":["Image tensors (variable resolution, extracted from PDF pages)","Text strings (OCR output and document text)","Structured JSON with image-text alignment metadata","WebDataset tar archives for distributed training"],"categories":["data-processing-analysis","multimodal-dataset"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_1","uri":"capability://data.processing.analysis.streaming.access.to.large.scale.multimodal.samples.via.webdataset.format","name":"streaming access to large-scale multimodal samples via webdataset format","description":"Implements WebDataset tar-based streaming protocol that allows sequential access to image-text pairs without downloading the entire 633K-sample dataset. Uses tar archive sharding and lazy loading to enable training on machines with limited disk space, with built-in support for distributed data loading across multiple GPUs/TPUs via HuggingFace datasets library integration.","intents":["Train models on full dataset without requiring 500GB+ local storage","Implement distributed training across multiple machines with efficient data shuffling","Stream data directly from HuggingFace Hub with automatic caching and resumption","Reduce training startup time by avoiding full dataset download before training begins"],"best_for":["Teams with limited GPU memory or storage running large-scale training jobs","Researchers using cloud infrastructure (AWS, GCP, Azure) with per-GB egress costs","Distributed training setups requiring efficient multi-worker data loading","Rapid prototyping scenarios where full dataset download is prohibitive"],"limitations":["Sequential tar streaming introduces ~5-15% throughput overhead vs pre-extracted image files due to decompression","Random access to specific samples is inefficient; requires linear scan through tar archives","Network latency becomes bottleneck for training on slow connections; requires >10Mbps sustained bandwidth","Tar archive boundaries may not align with training epoch boundaries, requiring custom epoch handling logic"],"requires":["HuggingFace datasets library (>=2.14.0) with WebDataset backend support","WebDataset Python library (>=0.2.0)","Stable internet connection with >10Mbps bandwidth for streaming","PyTorch DataLoader or equivalent with custom collate function for tar sample unpacking"],"input_types":["WebDataset tar archives (sharded across multiple files)","HuggingFace Hub dataset identifiers"],"output_types":["Batched image tensors and text strings","Lazy-loaded samples with on-demand decompression","Distributed data loader iterators compatible with PyTorch training loops"],"categories":["data-processing-analysis","automation-workflow"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_2","uri":"capability://data.processing.analysis.reproducible.dataset.versioning.and.metadata.discovery.via.mlcroissant.standard","name":"reproducible dataset versioning and metadata discovery via mlcroissant standard","description":"Encodes dataset structure, provenance, and licensing metadata in MLCroissant format, enabling automated discovery, citation, and reproducible dataset loading across different tools and frameworks. Metadata includes source URLs, extraction timestamps, license information (CC-BY-4.0), and data schema definitions that allow downstream tools to validate data integrity and understand dataset composition without manual inspection.","intents":["Discover and cite the dataset in research papers with standardized metadata","Validate dataset integrity and schema compliance before training","Reproduce exact dataset versions across different research teams and time periods","Automatically generate data documentation and schema validation rules"],"best_for":["Researchers publishing papers requiring reproducible dataset specifications","Organizations implementing data governance and compliance tracking","Teams building automated data pipeline validation systems","ML platforms integrating multiple datasets with standardized metadata"],"limitations":["MLCroissant standard is relatively new (2023-2024); limited tooling ecosystem compared to established metadata formats","Metadata does not include per-sample quality scores or filtering criteria — users must implement own quality assessment","Version pinning requires HuggingFace Hub infrastructure; offline reproducibility requires local metadata snapshots","Metadata schema does not capture dynamic properties like data drift or distribution shifts over time"],"requires":["MLCroissant-compatible metadata reader (available in HuggingFace datasets library >=2.14.0)","JSON schema validation tools for metadata compliance checking","Access to HuggingFace Hub for metadata retrieval and versioning"],"input_types":["MLCroissant JSON metadata files","Dataset identifiers (mlfoundations/MINT-1T-PDF-CC-2023-23)"],"output_types":["Structured metadata JSON with schema definitions","Citation information and license terms","Data validation reports and schema compliance checks"],"categories":["data-processing-analysis","memory-knowledge"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_3","uri":"capability://image.visual.pdf.native.image.text.alignment.extraction.with.layout.preservation","name":"pdf-native image-text alignment extraction with layout preservation","description":"Extracts image-text pairs from PDF documents while preserving spatial layout information, semantic relationships, and document structure (e.g., captions near figures, text flowing around images). Uses PDF parsing to identify image boundaries and associated text blocks, maintaining coordinate information that enables downstream tasks like layout understanding and spatial reasoning without requiring separate OCR or layout analysis steps.","intents":["Train models that understand document layout and spatial relationships between images and text","Build systems that can answer questions about figure locations and associated captions","Create datasets for table extraction, diagram understanding, and document structure analysis","Develop layout-aware retrieval systems that understand document semantics beyond text content"],"best_for":["Researchers building document understanding and layout analysis models","Teams developing enterprise document processing systems","Organizations training models for scientific paper analysis and figure extraction","Builders creating document-aware RAG systems that leverage layout context"],"limitations":["Extraction quality depends on PDF structure; poorly formatted or scanned PDFs may have misaligned image-text pairs","Layout coordinates are PDF-specific; conversion to normalized formats requires additional processing","No built-in handling of multi-column layouts, text wrapping, or complex document structures","Extracted text may contain OCR errors from scanned PDFs; no quality filtering or confidence scores provided","Layout information is not standardized across different PDF creation tools; downstream models must handle format variations"],"requires":["PDF parsing library compatible with Common Crawl PDFs (e.g., pdfplumber, PyPDF2)","Python 3.8+","HuggingFace datasets library for dataset access"],"input_types":["PDF documents from Common Crawl 2023","PDF metadata and structure information"],"output_types":["Image tensors extracted from PDF pages","Associated text strings with spatial coordinates","Layout metadata (bounding boxes, text flow direction, document structure)","JSON with image-text alignment information"],"categories":["image-visual","data-processing-analysis"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_4","uri":"capability://data.processing.analysis.common.crawl.2023.pdf.document.filtering.and.quality.curation","name":"common crawl 2023 pdf document filtering and quality curation","description":"Filters and curates 1T+ tokens of PDF documents from Common Crawl 2023 snapshot using quality heuristics (document completeness, text-image ratio, language detection, format validity) to create a high-quality subset of 633K samples. Implements multi-stage filtering pipeline that removes corrupted PDFs, non-English content, and documents with poor image-text alignment, producing a dataset suitable for training vision-language models without extensive downstream cleaning.","intents":["Access curated, high-quality PDF data without manual filtering and quality assessment","Train models on diverse document types from web-scale Common Crawl without data quality issues","Understand filtering criteria and quality thresholds used for dataset curation","Reproduce filtering pipeline for custom dataset creation from Common Crawl"],"best_for":["ML teams training vision-language models who want pre-filtered, production-ready data","Researchers studying document understanding on web-scale data","Organizations building document processing systems that need diverse document types","Teams implementing custom filtering pipelines based on published criteria"],"limitations":["Filtering criteria are not fully documented; exact thresholds and heuristics are not publicly specified","Curation is one-time snapshot from 2023; does not include newer documents or evolving document types","Quality filtering may introduce bias toward certain document types (e.g., academic papers, technical documentation) and exclude others","No per-sample quality scores or confidence metrics; users cannot adjust filtering thresholds for their use cases","Filtering pipeline is not reproducible without access to original Common Crawl infrastructure and filtering code"],"requires":["Understanding of Common Crawl 2023 snapshot structure","HuggingFace datasets library for accessing curated dataset","No direct access to filtering pipeline required; dataset is pre-curated"],"input_types":["Common Crawl 2023 PDF documents (1T+ tokens)","Quality filtering heuristics and thresholds"],"output_types":["Filtered subset of 633K high-quality PDF samples","Metadata about filtering decisions and quality metrics","Curated image-text pairs with quality guarantees"],"categories":["data-processing-analysis"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_5","uri":"capability://data.processing.analysis.english.language.document.filtering.and.multilingual.dataset.composition","name":"english-language document filtering and multilingual dataset composition","description":"Filters dataset to English-language documents using language detection heuristics applied during curation, ensuring consistent language composition for training English-focused vision-language models. Implements language identification at document and sample level, removing non-English PDFs and mixed-language content to maintain dataset homogeneity and training stability.","intents":["Train English-language vision-language models without multilingual interference","Ensure consistent language composition across training batches","Understand language distribution and filtering criteria in the dataset","Build English-specific document understanding systems without multilingual complexity"],"best_for":["Teams training English-focused vision-language models","Researchers building document understanding systems for English documents","Organizations developing English-language document processing pipelines","Builders creating English-specific RAG and document retrieval systems"],"limitations":["Language filtering excludes non-English documents; not suitable for multilingual model training","Language detection heuristics may misclassify mixed-language documents or documents with code/technical content","No per-sample language confidence scores; users cannot adjust language filtering thresholds","Filtering criteria and language detection method are not fully documented","Dataset composition may not reflect true language distribution of Common Crawl; filtering introduces bias toward English-heavy document types"],"requires":["Language detection library (e.g., langdetect, textblob) for understanding filtering approach","HuggingFace datasets library for accessing filtered dataset"],"input_types":["PDF documents with text content","Language detection heuristics"],"output_types":["English-language filtered dataset (633K samples)","Language metadata and filtering decisions","Homogeneous language composition for training"],"categories":["data-processing-analysis"],"confidence":0.5,"matches":0,"success_rate":0},{"id":"hf-dataset-mlfoundations--mint-1t-pdf-cc-2023-23__cap_6","uri":"capability://data.processing.analysis.cc.by.4.0.licensed.dataset.with.attribution.and.commercial.use.compliance","name":"cc-by-4.0 licensed dataset with attribution and commercial use compliance","description":"Dataset is released under Creative Commons Attribution 4.0 (CC-BY-4.0) license, enabling commercial use with attribution requirements. License metadata is embedded in MLCroissant format and HuggingFace Hub, providing clear terms for usage, redistribution, and derivative works. Requires attribution to original sources and compliance with underlying Common Crawl and source document licenses.","intents":["Use dataset for commercial model training with clear legal compliance","Understand attribution requirements and licensing terms before deployment","Redistribute or create derivative datasets with proper license compliance","Ensure legal compliance in production systems using trained models"],"best_for":["Commercial organizations training models for production deployment","Teams building products that require clear IP and licensing terms","Researchers publishing work with commercial applications","Organizations with strict compliance and legal requirements"],"limitations":["CC-BY-4.0 requires attribution to original sources; failure to attribute may result in license violation","Underlying Common Crawl documents may have additional license restrictions not captured in CC-BY-4.0","Source documents may have been created under different licenses; compliance requires checking original source licenses","Commercial use requires attribution in deployed systems; unclear how to attribute in production models","License does not provide indemnification or warranty; users assume all legal risk"],"requires":["Understanding of CC-BY-4.0 license terms and attribution requirements","Legal review for commercial use cases","Compliance tracking for attribution in deployed systems"],"input_types":["CC-BY-4.0 license terms","MLCroissant metadata with license information"],"output_types":["License compliance documentation","Attribution requirements and source information","Legal terms for commercial use"],"categories":["data-processing-analysis"],"confidence":0.5,"matches":0,"success_rate":0}],"trust":{"score":24,"verified":false,"data_access_risk":"high","permissions":["Python 3.8+","HuggingFace datasets library (>=2.14.0)","WebDataset library (>=0.2.0) for efficient streaming","~500GB+ disk space for full dataset download or streaming access via HuggingFace Hub","MLCroissant-compatible metadata reader for reproducible dataset versioning","HuggingFace datasets library (>=2.14.0) with WebDataset backend support","WebDataset Python library (>=0.2.0)","Stable internet connection with >10Mbps bandwidth for streaming","PyTorch DataLoader or equivalent with custom collate function for tar sample unpacking","MLCroissant-compatible metadata reader (available in HuggingFace datasets library >=2.14.0)"],"failure_modes":["633K samples represent filtered subset of Common Crawl — not exhaustive coverage of all document types or languages","PDF extraction quality depends on source document structure; scanned/image-based PDFs may have degraded text alignment","WebDataset streaming format requires compatible data loading libraries; not directly compatible with standard PyTorch DataLoader without adapters","CC-BY-4.0 license requires attribution; commercial use requires compliance with original source licenses","No built-in deduplication or quality filtering — downstream users must implement their own data cleaning pipelines","Sequential tar streaming introduces ~5-15% throughput overhead vs pre-extracted image files due to decompression","Random access to specific samples is inefficient; requires linear scan through tar archives","Network latency becomes bottleneck for training on slow connections; requires >10Mbps sustained bandwidth","Tar archive boundaries may not align with training epoch boundaries, requiring custom epoch handling logic","MLCroissant standard is relatively new (2023-2024); limited tooling ecosystem compared to established metadata formats","builder identity is not verified yet","no observed match outcomes yet"],"rank_breakdown":{"adoption":0.05,"quality":0.24,"ecosystem":0.5000000000000001,"match_graph":0.25,"freshness":0.75,"weights":{"adoption":0.3,"quality":0.25,"ecosystem":0.1,"match_graph":0.3,"freshness":0.05}},"observed_outcomes":{"matches":0,"success_rate":0,"avg_confidence":0,"top_intents":[],"last_matched_at":null},"maintenance":{"status":"active","updated_at":"2026-05-24T12:16:22.764Z","last_scraped_at":"2026-04-22T08:08:14.361Z","last_commit":null},"community":{"stars":null,"forks":null,"weekly_downloads":null,"model_downloads":null,"model_likes":null}},"distribution":{"claim_url":"https://unfragile.ai/submit?claim=mlfoundations--mint-1t-pdf-cc-2023-23","compare_url":"https://unfragile.ai/compare?artifact=mlfoundations--mint-1t-pdf-cc-2023-23"}},"signature":"ckG1H6bOj91b8ZIilSVuYmkEU8scEE84amKD038o38astlQ0rWW9+NH42MlD/wAWDhbSJxmPiKudhRjJrLjgCg==","signedAt":"2026-06-22T13:59:31.696Z","signedBy":"unfragile.ai","version":1},"_links":{"self":"https://unfragile.ai/api/v1/passport/mlfoundations--mint-1t-pdf-cc-2023-23","artifact":"https://unfragile.ai/mlfoundations--mint-1t-pdf-cc-2023-23","verify":"https://unfragile.ai/api/v1/verify?slug=mlfoundations--mint-1t-pdf-cc-2023-23","publicKey":"https://unfragile.ai/api/v1/trust-passport-public-key","spec":"https://unfragile.ai/trust","schema":"https://unfragile.ai/schema.json","docs":"https://unfragile.ai/docs"}}