{"authors":[{"id":null,"fullName":"Eleni Straitouri","name":null,"surname":null,"rank":1,"pid":null},{"id":null,"fullName":"Lequn Wang","name":null,"surname":null,"rank":2,"pid":null},{"id":null,"fullName":"Nastaran Okati","name":null,"surname":null,"rank":3,"pid":null},{"id":null,"fullName":"Manuel Gomez Rodriguez","name":null,"surname":null,"rank":4,"pid":null}],"openAccessColor":null,"publiclyFunded":false,"eoscIfGuidelines":null,"type":"publication","language":{"code":"und","label":"Undetermined"},"countries":null,"subjects":[{"subject":{"scheme":"keyword","value":"FOS: Computer and information sciences"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Science - Machine Learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Science - Computers and Society"},"provenance":null},{"subject":{"scheme":"keyword","value":"Statistics - Machine Learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computers and Society (cs.CY)"},"provenance":null},{"subject":{"scheme":"FOS","value":"0202 electrical engineering, electronic engineering, information engineering"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Science - Human-Computer Interaction"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (stat.ML)"},"provenance":null},{"subject":{"scheme":"FOS","value":"02 engineering and technology"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (cs.LG)"},"provenance":null},{"subject":{"scheme":"keyword","value":"Human-Computer Interaction (cs.HC)"},"provenance":null}],"mainTitle":"Improving Expert Predictions with Conformal Prediction","subTitle":null,"descriptions":["Automated decision support systems promise to help human experts solve multiclass classification tasks more efficiently and accurately. However, existing systems typically require experts to understand when to cede agency to the system or when to exercise their own agency. Otherwise, the experts may be better off solving the classification tasks on their own. In this work, we develop an automated decision support system that, by design, does not require experts to understand when to trust the system to improve performance. Rather than providing (single) label predictions and letting experts decide when to trust these predictions, our system provides sets of label predictions constructed using conformal prediction$\\unicode{x2014}$prediction sets$\\unicode{x2014}$and forcefully asks experts to predict labels from these sets. By using conformal prediction, our system can precisely trade-off the probability that the true label is not in the prediction set, which determines how frequently our system will mislead the experts, and the size of the prediction set, which determines the difficulty of the classification task the experts need to solve using our system. In addition, we develop an efficient and near-optimal search method to find the conformal predictor under which the experts benefit the most from using our system. Simulation experiments using synthetic and real expert predictions demonstrate that our system may help experts make more accurate predictions and is robust to the accuracy of the classifier the conformal predictor relies on.","Published at ICML 2023"],"publicationDate":"2022-01-01","publisher":null,"embargoEndDate":"2022-01-01","sources":null,"formats":null,"contributors":null,"coverages":null,"bestAccessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":null,"documentationUrls":null,"codeRepositoryUrl":null,"programmingLanguage":null,"contactPeople":null,"contactGroups":null,"tools":null,"size":null,"version":null,"geoLocations":null,"id":"doi_dedup___::6f6d27d9afe551bcdb470f8d887ce9fd","originalIds":["50|datacite____::6f6d27d9afe551bcdb470f8d887ce9fd","10.48550/arxiv.2201.12006","oai:arXiv.org:2201.12006","50|od________18::0459a732b228fec9ed3db6826f9fe9fd","50|dblp________::69bf8662ae5df11248aeeb7739a50790"],"pids":[{"scheme":"doi","value":"10.48550/arxiv.2201.12006"},{"scheme":"arXiv","value":"2201.12006"}],"dateOfCollection":null,"lastUpdateTimeStamp":null,"indicators":{"citationImpact":{"citationCount":0.0,"influence":2.1746283E-9,"popularity":1.4615026E-9,"impulse":0.0,"citationClass":"C5","influenceClass":"C5","impulseClass":"C5","popularityClass":"C5"}},"projects":[{"id":"corda__h2020::83d6aa753c4b6e94582f4ef733616892","code":"945719","acronym":"HumanML","title":"Human-Centric Machine Learning","funder":"European Commission","pids":[{"scheme":"doi","value":"10.3030/945719"}]},{"id":"nsf_________::43aafd2fffd5c406dc6849873673c95f","code":"1901168","acronym":null,"title":"III: Medium: Collaborative Research: Counterfactual Learning and Evaluation for Interactive Information Systems","funder":"National Science Foundation","pids":null},{"id":"nsf_________::be6f5d34d405f119140a67c362587f3e","code":"2008139","acronym":null,"title":"III: Small: Fairness and Control of Exposure in Ranking","funder":"National Science Foundation","pids":null}],"organizations":[{"legalName":"Max Planck Society","acronym":"MPG","id":"openorgs____::5a405a89387d1881afa956f475994e10","pids":[{"scheme":"ISNI","value":"0000000121051091"},{"scheme":"ROR","value":"https://ror.org/01hhn8329"},{"scheme":"GRID","value":"grid.4372.2"},{"scheme":"PIC","value":"999990267"},{"scheme":"RRID","value":"RRID:nlx_55250"},{"scheme":"FundRef","value":"501100004189"},{"scheme":"RRID","value":"RRID:SCR_011366"},{"scheme":"Wikidata","value":"Q158085"},{"scheme":"OrgRef","value":"614753"},{"scheme":"mag_id","value":"149899117"},{"scheme":"wikidata","value":"Q158085"},{"scheme":"fundref","value":"501100004189"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"http://www.mpg.de/en"},{"legalName":"Cornell University","acronym":"CU","id":"openorgs____::c3b051f702ed16eb249c25e45abd85d3","pids":[{"scheme":"FundRef","value":"100008674"},{"scheme":"RRID","value":"RRID:SCR_003998"},{"scheme":"fundref","value":"100008585"},{"scheme":"FundRef","value":"100006480"},{"scheme":"fundref","value":"100009083"},{"scheme":"mag_id","value":"205783295"},{"scheme":"FundRef","value":"100006077"},{"scheme":"Wikidata","value":"Q49115"},{"scheme":"FundRef","value":"100007231"},{"scheme":"fundref","value":"100013642"},{"scheme":"FundRef","value":"100006923"},{"scheme":"fundref","value":"100006480"},{"scheme":"FundRef","value":"100021772"},{"scheme":"fundref","value":"100006077"},{"scheme":"FundRef","value":"100007272"},{"scheme":"PIC","value":"999836134"},{"scheme":"fundref","value":"100006471"},{"scheme":"wikidata","value":"Q49115"},{"scheme":"ISNI","value":"000000041936877X"},{"scheme":"FundRef","value":"100007273"},{"scheme":"RRID","value":"RRID:nlx_58666"},{"scheme":"fundref","value":"100015376"},{"scheme":"FundRef","value":"100013233"},{"scheme":"fundref","value":"100021772"},{"scheme":"fundref","value":"100008586"},{"scheme":"OrgRef","value":"2551571"},{"scheme":"ROR","value":"https://ror.org/05bnh6r87"},{"scheme":"FundRef","value":"100015376"},{"scheme":"fundref","value":"100007272"},{"scheme":"FundRef","value":"100008585"},{"scheme":"FundRef","value":"100008586"},{"scheme":"fundref","value":"100007231"},{"scheme":"FundRef","value":"100013642"},{"scheme":"FundRef","value":"100007232"},{"scheme":"FundRef","value":"100009083"},{"scheme":"fundref","value":"100013976"},{"scheme":"fundref","value":"100006923"},{"scheme":"fundref","value":"100013233"},{"scheme":"FundRef","value":"100007604"},{"scheme":"fundref","value":"100008674"},{"scheme":"FundRef","value":"100006471"},{"scheme":"GRID","value":"grid.5386.8"},{"scheme":"FundRef","value":"100013976"},{"scheme":"OrgRef","value":"7954422"},{"scheme":"fundref","value":"100007232"},{"scheme":"OrgRef","value":"329738"}],"countries":[{"code":"US","label":"United States"}],"websiteurl":"http://www.cornell.edu/"},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::23e60f4ea4baa2657664c56f74f86459","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::0a0ea8130e02dec257603d13acc8f37b","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Max Planck Institute for Software Systems","acronym":"MPI-SWS","id":"openorgs____::a6be3b23fdb0e61554b444ec8c1e8d1c","pids":[{"scheme":"Wikidata","value":"Q881238"},{"scheme":"GRID","value":"grid.469860.5"},{"scheme":"ISNI","value":"000000040492020X"},{"scheme":"ROR","value":"https://ror.org/02pe2kf23"},{"scheme":"wikidata","value":"Q881238"}],"countries":[{"code":"DE","label":"Germany"}],"websiteurl":"https://www.mpi-sws.org/index.php"},{"legalName":"CORNELL UNIVERSITY","acronym":"CORNELL UNIVERSITY","id":"pending_org_::b45be9b102f9c6ef73e325f0adef8233","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"CORNELL UNIVERSITY","acronym":"CORNELL UNIVERSITY","id":"pending_org_::4df607cb62aa7912379c179f999a97fd","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::da169c3d72537e21622b1fbb5109ca2f","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::a2ae5dab283ca33263b24b41087b8842","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::3d13968ea9ea2f6eb9b0fa351f24a12c","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::71746d9abbfa0c3d9879b4f3db2ed19e","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"CORNELL UNIVERSITY","acronym":"CORNELL UNIVERSITY","id":"pending_org_::4336a783417e05853d60770d8ca28415","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::03bf571dba8a52adc1a660df75f6aa23","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null},{"legalName":"Cornell University","acronym":"Cornell University","id":"pending_org_::f36a2aa59aa2127c478859bccafdb6ff","pids":null,"countries":[{"code":"UNKNOWN","label":"Unknown"}],"websiteurl":null}],"communities":null,"collectedFrom":[{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}],"instances":[{"pids":[{"scheme":"doi","value":"10.48550/arxiv.2201.12006"}],"license":"CC BY","type":"Article","urls":["https://dx.doi.org/10.48550/arxiv.2201.12006"],"publicationDate":"2022-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}},{"pids":[{"scheme":"arXiv","value":"2201.12006"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Preprint","urls":["http://arxiv.org/abs/2201.12006"],"publicationDate":"2022-01-28","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},"collectedFrom":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}},{"type":"Conference object","urls":["https://proceedings.mlr.press/v202/straitouri23a.html","https://dblp.org/rec/conf/icml/StraitouriWOR23.html"],"refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}}],"links":[{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"corda__h2020::83d6aa753c4b6e94582f4ef733616892","relatedRecordType":"project","relationProvenance":"iis","trust":"0.897"},"collectedfrom":[{"dsId":"openaire____::a55eb91348674d853191f4f4fd73d078","dsName":"CORDA - COmmon Research DAta Warehouse - Horizon 2020"}],"projectTitle":"Human-Centric Machine Learning","code":"945719","funding":{"funder":{"id":"ec__________::EC","shortname":"EC","name":"European Commission","jurisdiction":{"code":"EU","label":"European Union"},"pid":null},"level0":{"id":"ec__________::EC::H2020","description":"Horizon 2020 Framework Programme","name":"H2020"},"level1":{"id":"ec__________::EC::H2020::ERC","description":"European Research Council","name":"ERC"},"level2":{"id":"ec__________::EC::H2020::ERC::ERC-STG","description":"Starting Grant","name":"ERC-STG"}},"startDate":"2020-11-01","endDate":"2025-10-31"},{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"nsf_________::43aafd2fffd5c406dc6849873673c95f","relatedRecordType":"project","relationProvenance":"iis","trust":"0.7348"},"collectedfrom":[{"dsId":"openaire____::dd69b4a1513c9de9f46faf24048da1e8","dsName":"NSF - National Science Foundation"}],"projectTitle":"III: Medium: Collaborative Research: Counterfactual Learning and Evaluation for Interactive Information Systems","code":"1901168","funding":{"funder":{"id":"nsf_________::NSF","shortname":"NSF","name":"National Science Foundation","jurisdiction":{"code":"US","label":"United States"},"pid":null},"level0":{"id":"nsf_________::NSF::CISE/OAD","description":"Directorate for Computer & Information Science & Engineering","name":"Directorate for Computer & Information Science & Engineering"},"level1":{"id":"nsf_________::NSF::CISE/OAD::CISE/IIS","description":"Division of Information and Intelligent Systems","name":"Division of Information and Intelligent Systems"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2019-08-15","endDate":"2023-07-31"},{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"nsf_________::be6f5d34d405f119140a67c362587f3e","relatedRecordType":"project","relationProvenance":"iis","trust":"0.7348"},"collectedfrom":[{"dsId":"openaire____::dd69b4a1513c9de9f46faf24048da1e8","dsName":"NSF - National Science Foundation"}],"projectTitle":"III: Small: Fairness and Control of Exposure in Ranking","code":"2008139","funding":{"funder":{"id":"nsf_________::NSF","shortname":"NSF","name":"National Science Foundation","jurisdiction":{"code":"US","label":"United States"},"pid":null},"level0":{"id":"nsf_________::NSF::CISE/OAD","description":"Directorate for Computer & Information Science & Engineering","name":"Directorate for Computer & Information Science & Engineering"},"level1":{"id":"nsf_________::NSF::CISE/OAD::CISE/IIS","description":"Division of Information and Intelligent Systems","name":"Division of Information and Intelligent Systems"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2020-10-01","endDate":"2023-09-30"}],"otherTitles":null,"inDiamondJournal":false,"green":true,"isGreen":true,"isInDiamondJournal":false}