{"authors":[{"id":"orcid_______::8b3d6a64678d9a185f6aafcc22ba7dd1","fullName":"Xiao Fu","name":null,"surname":null,"rank":1,"pid":{"id":{"scheme":"orcid","value":"0000-0002-3321-8167"},"provenance":null}},{"id":null,"fullName":"Shangzhan Zhang","name":null,"surname":null,"rank":2,"pid":{"id":{"scheme":"orcid_pending","value":"0009-0006-4693-6009"},"provenance":null}},{"id":"orcid_______::7360482ec1f0b85d5ead9e375f99c50c","fullName":"Tianrun Chen","name":null,"surname":null,"rank":3,"pid":{"id":{"scheme":"orcid","value":"0000-0003-0177-0157"},"provenance":null}},{"id":"orcid_______::e9a64fce321f3c1abb809f47a44e7c2b","fullName":"Yichong Lu","name":null,"surname":null,"rank":4,"pid":{"id":{"scheme":"orcid","value":"0009-0007-3246-0166"},"provenance":null}},{"id":null,"fullName":"Xiaowei Zhou 0001","name":null,"surname":null,"rank":5,"pid":{"id":{"scheme":"orcid_pending","value":"0000-0003-1926-5597"},"provenance":null}},{"id":"orcid_______::e81a8cf891bba4e58faab0acbe1a729b","fullName":"Andreas Geiger 0001","name":null,"surname":null,"rank":6,"pid":{"id":{"scheme":"orcid","value":"0000-0002-8151-3726"},"provenance":null}},{"id":null,"fullName":"Yiyi Liao","name":null,"surname":null,"rank":7,"pid":{"id":{"scheme":"orcid_pending","value":"0000-0001-6662-3022"},"provenance":null}}],"openAccessColor":null,"publiclyFunded":false,"eoscIfGuidelines":null,"type":"publication","language":{"code":"und","label":"Undetermined"},"countries":null,"subjects":[{"subject":{"scheme":"keyword","value":"FOS: Computer and information sciences"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Vision and Pattern Recognition (cs.CV)"},"provenance":null},{"subject":{"scheme":"FOS","value":"0202 electrical engineering, electronic engineering, information engineering"},"provenance":null},{"subject":{"scheme":"FOS","value":"02 engineering and technology"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Vision and Pattern Recognition"},"provenance":null}],"mainTitle":"PanopticNeRF-360: Panoramic 3D-to-2D Label Transfer in Urban Scenes","subTitle":null,"descriptions":["Training perception systems for self-driving cars requires substantial 2D annotations that are labor-intensive to manual label. While existing datasets provide rich annotations on pre-recorded sequences, they fall short in labeling rarely encountered viewpoints, potentially hampering the generalization ability for perception models. In this paper, we present PanopticNeRF-360, a novel approach that combines coarse 3D annotations with noisy 2D semantic cues to generate high-quality panoptic labels and images from any viewpoint. Our key insight lies in exploiting the complementarity of 3D and 2D priors to mutually enhance geometry and semantics. Specifically, we propose to leverage coarse 3D bounding primitives and noisy 2D semantic and instance predictions to guide geometry optimization, by encouraging predicted labels to match panoptic pseudo ground truth. Simultaneously, the improved geometry assists in filtering 3D&2D annotation noise by fusing semantics in 3D space via a learned semantic field. To further enhance appearance, we combine MLP and hash grids to yield hybrid scene features, striking a balance between high-frequency appearance and contiguous semantics. Our experiments demonstrate PanopticNeRF-360's state-of-the-art performance over label transfer methods on the challenging urban scenes of the KITTI-360 dataset. Moreover, PanopticNeRF-360 enables omnidirectional rendering of high-fidelity, multi-view and spatiotemporally consistent appearance, semantic and instance labels. We make our code and data available at https://github.com/fuxiao0719/PanopticNeRF","TPAMI 2025. Project page: http://fuxiao0719.github.io/projects/panopticnerf360/ Code: https://github.com/fuxiao0719/PanopticNeRF/tree/panopticnerf360"],"publicationDate":"2025-10-01","publisher":"Institute of Electrical and Electronics Engineers (IEEE)","embargoEndDate":"2023-09-01","sources":["Crossref"],"formats":null,"contributors":null,"coverages":null,"bestAccessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issnPrinted":"0162-8828","issnOnline":"1939-3539","issnLinking":null,"ep":"8822","iss":null,"sp":"8804","vol":"47","edition":null,"conferencePlace":null,"conferenceDate":null},"documentationUrls":null,"codeRepositoryUrl":null,"programmingLanguage":null,"contactPeople":null,"contactGroups":null,"tools":null,"size":null,"version":null,"geoLocations":null,"id":"doi_dedup___::0f6b5ab8289f5dcdbedbcc11060d1fe6","originalIds":["10.1109/tpami.2025.3581608","50|doiboost____|52b7e755cfb5f1bb4204a923e038fa3d","50|datacite____::0f6b5ab8289f5dcdbedbcc11060d1fe6","10.48550/arxiv.2309.10815","40540381","50|od________18::4234e88b0349d71b26550ba8b60f93da","oai:arXiv.org:2309.10815","50|dblp________::e2aa37976436804e9eff23312d115abf","50|dblp________::f7db15cd505884eea4ed96b3a1a6602f"],"pids":[{"scheme":"doi","value":"10.1109/tpami.2025.3581608"},{"scheme":"doi","value":"10.48550/arxiv.2309.10815"},{"scheme":"pmid","value":"40540381"},{"scheme":"arXiv","value":"2309.10815"}],"dateOfCollection":null,"lastUpdateTimeStamp":null,"indicators":{"citationImpact":{"citationCount":0.0,"influence":2.1746283E-9,"popularity":2.2497804E-9,"impulse":0.0,"citationClass":"C5","influenceClass":"C5","impulseClass":"C5","popularityClass":"C5"}},"projects":[{"id":"corda__h2020::e483510cc4d36edf53bc908a5ec49d45","code":"850533","acronym":"LEGO-3D","title":"Learning Generative 3D Scene Models for Training and Validating Intelligent Systems","funder":"European Commission","pids":[{"scheme":"doi","value":"10.3030/850533"}]}],"organizations":[{"legalName":"EBERHARD KARLS UNIVERSITAET TUEBINGEN","acronym":"UT","id":"pending_org_::e0236607e1606414d6b350bc1ba582fd","pids":[{"scheme":"RRID","value":"RRID:SCR_011735"},{"scheme":"RRID","value":"RRID:nlx_155516"},{"scheme":"ROR","value":"https://ror.org/03a1kwz48"},{"scheme":"RRID","value":"RRID:SCR_016076"},{"scheme":"PIC","value":"999991916"},{"scheme":"OrgReg","value":"DE0008"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"http://www.uni-tuebingen.de"},{"legalName":"University of Tübingen","acronym":"University of Tübingen","id":"openorgs____::ca58585cd878d3df6de7871d6d5a6e9e","pids":[{"scheme":"mag_id","value":"8087733"},{"scheme":"ROR","value":"https://ror.org/03a1kwz48"},{"scheme":"fundref","value":"501100002346"},{"scheme":"FundRef","value":"501100002346"},{"scheme":"fundref","value":"501100002345"},{"scheme":"FundRef","value":"501100009397"},{"scheme":"GRID","value":"grid.10392.39"},{"scheme":"OrgRef","value":"262301"},{"scheme":"FundRef","value":"501100002345"},{"scheme":"ISNI","value":"0000000121901447"},{"scheme":"Wikidata","value":"Q153978"},{"scheme":"fundref","value":"501100009397"},{"scheme":"wikidata","value":"Q153978"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"https://www.uni-tuebingen.de/en/university.html"},{"legalName":"Zhejiang Ocean University","acronym":"ZJOU","id":"openorgs____::0675420e957794c8d7610fde78a911ba","pids":[{"scheme":"GRID","value":"grid.413073.2"},{"scheme":"fundref","value":"501100008843"},{"scheme":"Wikidata","value":"Q8070661"},{"scheme":"OrgRef","value":"21491793"},{"scheme":"mag_id","value":"76130692"},{"scheme":"OrgRef","value":"24448753"},{"scheme":"Wikidata","value":"Q41715113"},{"scheme":"FundRef","value":"501100002771"},{"scheme":"fundref","value":"501100005356"},{"scheme":"fundref","value":"100009109"},{"scheme":"wikidata","value":"Q8070661"},{"scheme":"FundRef","value":"501100008843"},{"scheme":"Wikidata","value":"Q197543"},{"scheme":"FundRef","value":"501100004835"},{"scheme":"ISNI","value":"000000041759700X"},{"scheme":"GRID","value":"grid.13402.34"},{"scheme":"fundref","value":"501100002928"},{"scheme":"OrgRef","value":"340391"},{"scheme":"fundref","value":"501100002771"},{"scheme":"OrgRef","value":"41996598"},{"scheme":"ISNI","value":"0000000417589341"},{"scheme":"ROR","value":"https://ror.org/0331z5r71"},{"scheme":"FundRef","value":"501100002928"},{"scheme":"FundRef","value":"501100008827"},{"scheme":"wikidata","value":"Q197543"},{"scheme":"ROR","value":"https://ror.org/02vj4rn06"},{"scheme":"ISNI","value":"0000000091527385"},{"scheme":"mag_id","value":"1284762954"},{"scheme":"PIC","value":"957592440"},{"scheme":"ROR","value":"https://ror.org/00a2xv884"},{"scheme":"GRID","value":"grid.443483.c"},{"scheme":"fundref","value":"501100004835"},{"scheme":"mag_id","value":"31847773"},{"scheme":"OrgRef","value":"21501091"},{"scheme":"FundRef","value":"501100005356"},{"scheme":"Wikidata","value":"Q5967499"},{"scheme":"wikidata","value":"Q5967499"},{"scheme":"wikidata","value":"Q41715113"},{"scheme":"fundref","value":"501100008827"},{"scheme":"ISNI","value":"0000000418044247"},{"scheme":"ROR","value":"https://ror.org/03mys6533"},{"scheme":"GRID","value":"grid.443668.b"},{"scheme":"FundRef","value":"100009109"},{"scheme":"fundref","value":"501100012602"}],"countries":[{"code":"CN","label":"China (People's Republic of)"}],"websiteurl":"http://www.zjou.edu.cn/"},{"legalName":"Tübingen AI Center","acronym":"Tübingen AI Center","id":"pending_org_::22e730a41328a294bf521524962e8309","pids":[{"scheme":"ROR","value":"https://ror.org/0107nyd78"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"https://tuebingen.ai"}],"communities":null,"collectedFrom":[{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"},{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},{"key":"opendoar____::8b6dd7db9af49e67306feb59a8bdc52c","value":"Europe PubMed Central"},{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}],"instances":[{"pids":[{"scheme":"doi","value":"10.1109/tpami.2025.3581608"}],"license":"IEEE Copyright","accessRight":{"code":"c_14cb","label":"CLOSED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","urls":["https://doi.org/10.1109/tpami.2025.3581608"],"publicationDate":"2025-10-01","refereed":"peerReviewed","hostedBy":{"key":"issn___print::7ab2fec6925b36c35615919215ef0093","value":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},"collectedFrom":{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}},{"pids":[{"scheme":"doi","value":"10.48550/arxiv.2309.10815"}],"license":"arXiv Non-Exclusive Distribution","type":"Article","urls":["https://dx.doi.org/10.48550/arxiv.2309.10815"],"publicationDate":"2023-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}},{"pids":[{"scheme":"pmid","value":"40540381"}],"alternateIdentifiers":[{"scheme":"doi","value":"10.1109/tpami.2025.3581608"}],"type":"Article","urls":["https://pubmed.ncbi.nlm.nih.gov/40540381"],"refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"opendoar____::8b6dd7db9af49e67306feb59a8bdc52c","value":"Europe PubMed Central"}},{"pids":[{"scheme":"arXiv","value":"2309.10815"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Preprint","urls":["http://arxiv.org/abs/2309.10815"],"publicationDate":"2023-09-19","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},"collectedFrom":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2309.10815"}],"type":"Preprint","urls":["https://doi.org/10.48550/arXiv.2309.10815","https://dblp.org/rec/journals/corr/abs-2309-10815.html"],"publicationDate":"2023-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1109/tpami.2025.3581608"}],"type":"Article","urls":["https://doi.org/10.1109/TPAMI.2025.3581608","https://dblp.org/rec/journals/pami/FuZCLZGL25.html"],"publicationDate":"2025-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}}],"links":[{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"corda__h2020::e483510cc4d36edf53bc908a5ec49d45","relatedRecordType":"project","relationProvenance":"iis","trust":"0.897"},"collectedfrom":[{"dsId":"openaire____::a55eb91348674d853191f4f4fd73d078","dsName":"CORDA - COmmon Research DAta Warehouse - Horizon 2020"}],"projectTitle":"Learning Generative 3D Scene Models for Training and Validating Intelligent Systems","code":"850533","funding":{"funder":{"id":"ec__________::EC","shortname":"EC","name":"European Commission","jurisdiction":{"code":"EU","label":"European Union"},"pid":null},"level0":{"id":"ec__________::EC::H2020","description":"Horizon 2020 Framework Programme","name":"H2020"},"level1":{"id":"ec__________::EC::H2020::ERC","description":"European Research Council","name":"ERC"},"level2":{"id":"ec__________::EC::H2020::ERC::ERC-STG","description":"Starting Grant","name":"ERC-STG"}},"startDate":"2020-10-01","endDate":"2026-03-31"}],"otherTitles":null,"green":true,"inDiamondJournal":false,"isGreen":true,"isInDiamondJournal":false}