{"authors":[{"id":null,"fullName":"Wang, L.","name":"L.","surname":"Wang","rank":1,"pid":null},{"id":null,"fullName":"Joachims, T.","name":"T.","surname":"Joachims","rank":2,"pid":null},{"id":null,"fullName":"Gomez Rodriguez, M.","name":"M.","surname":"Gomez Rodriguez","rank":3,"pid":null}],"openAccessColor":null,"publiclyFunded":false,"eoscIfGuidelines":null,"type":"publication","language":{"code":"und","label":"Undetermined"},"countries":null,"subjects":[{"subject":{"scheme":"keyword","value":"FOS: Computer and information sciences"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Science - Machine Learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Science - Computers and Society"},"provenance":null},{"subject":{"scheme":"keyword","value":"Statistics - Machine Learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computers and Society (cs.CY)"},"provenance":null},{"subject":{"scheme":"FOS","value":"0202 electrical engineering, electronic engineering, information engineering"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (stat.ML)"},"provenance":null},{"subject":{"scheme":"FOS","value":"02 engineering and technology"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (cs.LG)"},"provenance":null}],"mainTitle":"Improving Screening Processes via Calibrated Subset Selection","subTitle":null,"descriptions":["Many selection processes such as finding patients qualifying for a medical trial or retrieval pipelines in search engines consist of multiple stages, where an initial screening stage focuses the resources on shortlisting the most promising candidates. In this paper, we investigate what guarantees a screening classifier can provide, independently of whether it is constructed manually or trained. We find that current solutions do not enjoy distribution-free theoretical guarantees -- we show that, in general, even for a perfectly calibrated classifier, there always exist specific pools of candidates for which its shortlist is suboptimal. Then, we develop a distribution-free screening algorithm -- called Calibrated Subset Selection (CSS) -- that, given any classifier and some amount of calibration data, finds near-optimal shortlists of candidates that contain a desired number of qualified candidates in expectation. Moreover, we show that a variant of CSS that calibrates a given classifier multiple times across specific groups can create shortlists with provable diversity guarantees. Experiments on US Census survey data validate our theoretical results and show that the shortlists provided by our algorithm are superior to those provided by several competitive baselines.","International Conference on Machine Learning (ICML) 2022"],"publicationDate":"2022-01-01","publisher":null,"embargoEndDate":"2022-02-01","sources":["Proceedings of the 39th International Conference on Machine Learning","Proceedings of Machine Learning Research"],"formats":["application/pdf"],"contributors":null,"coverages":null,"bestAccessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"CoRR","issnPrinted":null,"issnOnline":null,"issnLinking":null,"ep":null,"iss":null,"sp":null,"vol":"abs/2202.01147","edition":null,"conferencePlace":null,"conferenceDate":null},"documentationUrls":null,"codeRepositoryUrl":null,"programmingLanguage":null,"contactPeople":null,"contactGroups":null,"tools":null,"size":null,"version":null,"geoLocations":null,"id":"doi_dedup___::a5eeaec445522e5d7c39e31d034bf09e","originalIds":["50|datacite____::a5eeaec445522e5d7c39e31d034bf09e","10.48550/arxiv.2202.01147","oai:arXiv.org:2202.01147","50|od________18::a3375e82f8f2927387027ea63cd7c69e","50|dblp________::cf8955f37136aff4de623dd0395829e0","50|dblp________::e17e05a8b81d6bb47e31a2e497657cb5","oai:pure.mpg.de:item_3388399","50|od______1874::c27de229b4c14e21fc81efb83cb85a87","50|r3c4b2081b22::a5eeaec445522e5d7c39e31d034bf09e"],"pids":[{"scheme":"doi","value":"10.48550/arxiv.2202.01147"},{"scheme":"arXiv","value":"2202.01147"},{"scheme":"handle","value":"21.11116/0000-0011-AB4E-6"},{"scheme":"handle","value":"21.11116/0000-000A-9937-C"},{"scheme":"handle","value":"21.11116/0000-000A-9939-A"}],"dateOfCollection":null,"lastUpdateTimeStamp":null,"indicators":{"citationImpact":{"citationCount":0.0,"influence":2.1746283E-9,"popularity":1.4615026E-9,"impulse":0.0,"citationClass":"C5","influenceClass":"C5","impulseClass":"C5","popularityClass":"C5"}},"projects":[{"id":"corda__h2020::83d6aa753c4b6e94582f4ef733616892","code":"945719","acronym":"HumanML","title":"Human-Centric Machine Learning","funder":"European Commission","pids":[{"scheme":"doi","value":"10.3030/945719"}]},{"id":"nsf_________::43aafd2fffd5c406dc6849873673c95f","code":"1901168","acronym":null,"title":"III: Medium: Collaborative Research: Counterfactual Learning and Evaluation for Interactive Information Systems","funder":"National Science Foundation","pids":null},{"id":"nsf_________::be6f5d34d405f119140a67c362587f3e","code":"2008139","acronym":null,"title":"III: Small: Fairness and Control of Exposure in Ranking","funder":"National Science Foundation","pids":null}],"organizations":[{"legalName":"Cornell University","acronym":"CU","id":"openorgs____::c3b051f702ed16eb249c25e45abd85d3","pids":[{"scheme":"FundRef","value":"100008674"},{"scheme":"RRID","value":"RRID:SCR_003998"},{"scheme":"fundref","value":"100008585"},{"scheme":"FundRef","value":"100006480"},{"scheme":"fundref","value":"100009083"},{"scheme":"mag_id","value":"205783295"},{"scheme":"FundRef","value":"100006077"},{"scheme":"Wikidata","value":"Q49115"},{"scheme":"FundRef","value":"100007231"},{"scheme":"fundref","value":"100013642"},{"scheme":"FundRef","value":"100006923"},{"scheme":"fundref","value":"100006480"},{"scheme":"FundRef","value":"100021772"},{"scheme":"fundref","value":"100006077"},{"scheme":"FundRef","value":"100007272"},{"scheme":"PIC","value":"999836134"},{"scheme":"fundref","value":"100006471"},{"scheme":"wikidata","value":"Q49115"},{"scheme":"ISNI","value":"000000041936877X"},{"scheme":"FundRef","value":"100007273"},{"scheme":"RRID","value":"RRID:nlx_58666"},{"scheme":"fundref","value":"100015376"},{"scheme":"FundRef","value":"100013233"},{"scheme":"fundref","value":"100021772"},{"scheme":"fundref","value":"100008586"},{"scheme":"OrgRef","value":"2551571"},{"scheme":"ROR","value":"https://ror.org/05bnh6r87"},{"scheme":"FundRef","value":"100015376"},{"scheme":"fundref","value":"100007272"},{"scheme":"FundRef","value":"100008585"},{"scheme":"FundRef","value":"100008586"},{"scheme":"fundref","value":"100007231"},{"scheme":"FundRef","value":"100013642"},{"scheme":"FundRef","value":"100007232"},{"scheme":"FundRef","value":"100009083"},{"scheme":"fundref","value":"100013976"},{"scheme":"fundref","value":"100006923"},{"scheme":"fundref","value":"100013233"},{"scheme":"FundRef","value":"100007604"},{"scheme":"fundref","value":"100008674"},{"scheme":"FundRef","value":"100006471"},{"scheme":"GRID","value":"grid.5386.8"},{"scheme":"FundRef","value":"100013976"},{"scheme":"OrgRef","value":"7954422"},{"scheme":"fundref","value":"100007232"},{"scheme":"OrgRef","value":"329738"}],"countries":[{"code":"US","label":"United States"}],"websiteurl":"http://www.cornell.edu/"},{"legalName":"Max Planck Society","acronym":"MPG","id":"openorgs____::5a405a89387d1881afa956f475994e10","pids":[{"scheme":"ISNI","value":"0000000121051091"},{"scheme":"ROR","value":"https://ror.org/01hhn8329"},{"scheme":"GRID","value":"grid.4372.2"},{"scheme":"PIC","value":"999990267"},{"scheme":"RRID","value":"RRID:nlx_55250"},{"scheme":"FundRef","value":"501100004189"},{"scheme":"RRID","value":"RRID:SCR_011366"},{"scheme":"Wikidata","value":"Q158085"},{"scheme":"OrgRef","value":"614753"},{"scheme":"mag_id","value":"149899117"},{"scheme":"wikidata","value":"Q158085"},{"scheme":"fundref","value":"501100004189"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"http://www.mpg.de/en"},{"legalName":"CORNELL UNIVERSITY","acronym":"CORNELL UNIVERSITY","id":"pending_org_::4df607cb62aa7912379c179f999a97fd","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::a2ae5dab283ca33263b24b41087b8842","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::0a0ea8130e02dec257603d13acc8f37b","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::71746d9abbfa0c3d9879b4f3db2ed19e","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::23e60f4ea4baa2657664c56f74f86459","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University / Department of Information Science","acronym":"Cornell University / Department of Information Science","id":"pending_org_::cfb518239e28d41302627125fe491ef6","pids":null,"countries":[{"code":"US","label":"United States"}],"websiteurl":null},{"legalName":"Max Planck Institute for Software Systems","acronym":"MPI-SWS","id":"openorgs____::a6be3b23fdb0e61554b444ec8c1e8d1c","pids":[{"scheme":"Wikidata","value":"Q881238"},{"scheme":"GRID","value":"grid.469860.5"},{"scheme":"ISNI","value":"000000040492020X"},{"scheme":"ROR","value":"https://ror.org/02pe2kf23"},{"scheme":"wikidata","value":"Q881238"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"https://www.mpi-sws.org/index.php"},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::da169c3d72537e21622b1fbb5109ca2f","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"CORNELL UNIVERSITY","acronym":"CORNELL UNIVERSITY","id":"pending_org_::b45be9b102f9c6ef73e325f0adef8233","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"CORNELL UNIVERSITY","acronym":"CORNELL UNIVERSITY","id":"pending_org_::4336a783417e05853d60770d8ca28415","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::3d13968ea9ea2f6eb9b0fa351f24a12c","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::03bf571dba8a52adc1a660df75f6aa23","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::f36a2aa59aa2127c478859bccafdb6ff","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Department of Computer Science","acronym":"Department of Computer Science","id":"pending_org_::ae45e689ca9648f8bd97fe38b826798a","pids":null,"countries":[{"code":"ES","label":"Spain"}],"websiteurl":null}],"communities":null,"collectedFrom":[{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},{"key":"opendoar____::958adb57686c2fdec5796398de5f317a","value":"MPG.PuRe"},{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},{"key":"re3data_____::c4b2081b224be6b3e79d0e5e5556f631","value":"European Union Open Data Portal"}],"instances":[{"pids":[{"scheme":"doi","value":"10.48550/arxiv.2202.01147"}],"license":"arXiv Non-Exclusive Distribution","type":"Article","urls":["https://dx.doi.org/10.48550/arxiv.2202.01147"],"publicationDate":"2022-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}},{"pids":[{"scheme":"arXiv","value":"2202.01147"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Preprint","urls":["http://arxiv.org/abs/2202.01147"],"publicationDate":"2022-02-02","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},"collectedFrom":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}},{"alternateIdentifiers":[{"scheme":"arXiv","value":"2202.01147"}],"type":"Article","urls":["https://arxiv.org/abs/2202.01147","https://dblp.org/rec/journals/corr/abs-2202-01147.html"],"publicationDate":"2022-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"type":"Conference object","urls":["https://proceedings.mlr.press/v162/wang22j.html","https://dblp.org/rec/conf/icml/WangJR22.html"],"refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"pids":[{"scheme":"handle","value":"21.11116/0000-0011-AB4E-6"},{"scheme":"handle","value":"21.11116/0000-000A-9937-C"},{"scheme":"handle","value":"21.11116/0000-000A-9939-A"}],"alternateIdentifiers":[{"scheme":"urn","value":"https://proceedings.mlr.press/v162/wang22j.html"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Conference object","urls":["https://hdl.handle.net/21.11116/0000-000A-9937-C","https://hdl.handle.net/21.11116/0000-000A-9939-A","https://hdl.handle.net/21.11116/0000-0011-AB4E-6"],"publicationDate":"2022-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::958adb57686c2fdec5796398de5f317a","value":"MPG.PuRe"},"collectedFrom":{"key":"opendoar____::958adb57686c2fdec5796398de5f317a","value":"MPG.PuRe"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2202.01147"}],"type":"Conference object","urls":["http://dx.doi.org/10.48550/arxiv.2202.01147"],"publicationDate":"2022-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"re3data_____::c4b2081b224be6b3e79d0e5e5556f631","value":"European Union Open Data Portal"}}],"links":[{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"corda__h2020::83d6aa753c4b6e94582f4ef733616892","relatedRecordType":"project","relationProvenance":"sysimport:crosswalk:repository","trust":"0.9"},"collectedfrom":[{"dsId":"openaire____::a55eb91348674d853191f4f4fd73d078","dsName":"CORDA - COmmon Research DAta Warehouse - Horizon 2020"}],"projectTitle":"Human-Centric Machine Learning","code":"945719","funding":{"funder":{"id":"ec__________::EC","shortname":"EC","name":"European Commission","jurisdiction":{"code":"EU","label":"European Union"},"pid":null},"level0":{"id":"ec__________::EC::H2020","description":"Horizon 2020 Framework Programme","name":"H2020"},"level1":{"id":"ec__________::EC::H2020::ERC","description":"European Research Council","name":"ERC"},"level2":{"id":"ec__________::EC::H2020::ERC::ERC-STG","description":"Starting Grant","name":"ERC-STG"}},"startDate":"2020-11-01","endDate":"2025-10-31"},{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"nsf_________::43aafd2fffd5c406dc6849873673c95f","relatedRecordType":"project","relationProvenance":"iis","trust":"0.7348"},"collectedfrom":[{"dsId":"openaire____::dd69b4a1513c9de9f46faf24048da1e8","dsName":"NSF - National Science Foundation"}],"projectTitle":"III: Medium: Collaborative Research: Counterfactual Learning and Evaluation for Interactive Information Systems","code":"1901168","funding":{"funder":{"id":"nsf_________::NSF","shortname":"NSF","name":"National Science Foundation","jurisdiction":{"code":"US","label":"United States"},"pid":null},"level0":{"id":"nsf_________::NSF::CISE/OAD","description":"Directorate for Computer & Information Science & Engineering","name":"Directorate for Computer & Information Science & Engineering"},"level1":{"id":"nsf_________::NSF::CISE/OAD::CISE/IIS","description":"Division of Information and Intelligent Systems","name":"Division of Information and Intelligent Systems"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2019-08-15","endDate":"2023-07-31"},{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"nsf_________::be6f5d34d405f119140a67c362587f3e","relatedRecordType":"project","relationProvenance":"iis","trust":"0.7348"},"collectedfrom":[{"dsId":"openaire____::dd69b4a1513c9de9f46faf24048da1e8","dsName":"NSF - National Science Foundation"}],"projectTitle":"III: Small: Fairness and Control of Exposure in Ranking","code":"2008139","funding":{"funder":{"id":"nsf_________::NSF","shortname":"NSF","name":"National Science Foundation","jurisdiction":{"code":"US","label":"United States"},"pid":null},"level0":{"id":"nsf_________::NSF::CISE/OAD","description":"Directorate for Computer & Information Science & Engineering","name":"Directorate for Computer & Information Science & Engineering"},"level1":{"id":"nsf_________::NSF::CISE/OAD::CISE/IIS","description":"Division of Information and Intelligent Systems","name":"Division of Information and Intelligent Systems"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2020-10-01","endDate":"2023-09-30"}],"otherTitles":null,"green":true,"inDiamondJournal":false,"isGreen":true,"isInDiamondJournal":false}