{"authors":[{"id":"orcid_______::1cf57a6f502f62864632ac74501873b4","fullName":"Samuele Pasini","name":null,"surname":null,"rank":1,"pid":{"id":{"scheme":"orcid","value":"0000-0002-7900-3727"},"provenance":null}},{"id":null,"fullName":"Jinhan Kim","name":null,"surname":null,"rank":2,"pid":null},{"id":null,"fullName":"Tommaso Aiello","name":null,"surname":null,"rank":3,"pid":null},{"id":null,"fullName":"Rocío Cabrera Lozoya","name":null,"surname":null,"rank":4,"pid":null},{"id":null,"fullName":"Antonino Sabetta","name":null,"surname":null,"rank":5,"pid":null},{"id":null,"fullName":"Paolo Tonella","name":null,"surname":null,"rank":6,"pid":null}],"openAccessColor":"hybrid","publiclyFunded":false,"eoscIfGuidelines":null,"type":"publication","language":{"code":"eng","label":"English"},"countries":null,"subjects":[{"subject":{"scheme":"keyword","value":"FOS: Computer and information sciences"},"provenance":null},{"subject":{"scheme":"keyword","value":"Cryptography and Security"},"provenance":null},{"subject":{"scheme":"keyword","value":"Attack detection"},"provenance":null},{"subject":{"scheme":"keyword","value":"Software Engineering"},"provenance":null},{"subject":{"scheme":"FOS","value":"02 engineering and technology"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (cs.LG)"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"Software Engineering (cs.SE)"},"provenance":null},{"subject":{"scheme":"keyword","value":"Large Language Models"},"provenance":null},{"subject":{"scheme":"keyword","value":"Code security"},"provenance":null},{"subject":{"scheme":"keyword","value":"info:eu-repo/classification/udc/004"},"provenance":null},{"subject":{"scheme":"keyword","value":"Retrieval Augmented Generation"},"provenance":null},{"subject":{"scheme":"FOS","value":"0202 electrical engineering, electronic engineering, information engineering"},"provenance":null},{"subject":{"scheme":"keyword","value":"Cryptography and Security (cs.CR)"},"provenance":null}],"mainTitle":"Evaluating and improving the robustness of security attack detectors generated by LLMs","subTitle":null,"descriptions":["<jats:title>Abstract</jats:title>                   <jats:p>                     Large Language Models (LLMs) are increasingly used in software development to generate functions, such as                     <jats:italic>attack detectors</jats:italic>                     , that implement security requirements. A key challenge is ensuring the LLMs have enough knowledge to address specific security requirements, such as information about existing attacks. For this, we propose an approach integrating Retrieval Augmented Generation (RAG) and Self-Ranking into the LLM pipeline. RAG enhances the robustness of the output by incorporating external knowledge sources, while the Self-Ranking technique, inspired by the concept of Self-Consistency, generates multiple reasoning paths and creates ranks to select the most robust detector. Our extensive empirical study targets code generated by LLMs to detect two prevalent injection attacks in web security: Cross-Site Scripting (XSS) and SQL injection (SQLi). Results show a significant improvement in detection performance while employing RAG and Self-Ranking, with an increase of up to 71%pt (on average 37%pt) and up to 43%pt (on average 6%pt) in the F2-Score for XSS and SQLi detection, respectively.                   </jats:p>"],"publicationDate":"2025-12-05","publisher":"Springer Science and Business Media LLC","embargoEndDate":"2024-11-01","sources":["Crossref","Empirical Software Engineering. - 2025, vol. 31, no. 2, p. 35"],"formats":["application/pdf"],"contributors":null,"coverages":null,"bestAccessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"Empirical Software Engineering","issnPrinted":"1382-3256","issnOnline":"1573-7616","issnLinking":null,"ep":null,"iss":null,"sp":null,"vol":"31","edition":null,"conferencePlace":null,"conferenceDate":null},"documentationUrls":null,"codeRepositoryUrl":null,"programmingLanguage":null,"contactPeople":null,"contactGroups":null,"tools":null,"size":null,"version":null,"geoLocations":null,"id":"doi_dedup___::ada9d03d688259334b98100b7d8d43a1","originalIds":["10743","10.1007/s10664-025-10743-w","50|doiboost____|8434004b2756ec6bfb446b054cc8d8ed","50|datacite____::ada9d03d688259334b98100b7d8d43a1","10.48550/arxiv.2411.18216","oai:arXiv.org:2411.18216","50|od________18::8682852a8adf88aee9a2c4579e21ca99","50|dblp________::cc479909dcc088e617081b895a4b58e4","50|dblp________::d41589f76efa49ef1d875e54cd01f437","oai:sonar.ch:333995","50|od_____10363::beba7d1046a8259e41abc6925b939554"],"pids":[{"scheme":"doi","value":"10.1007/s10664-025-10743-w"},{"scheme":"doi","value":"10.48550/arxiv.2411.18216"},{"scheme":"arXiv","value":"2411.18216"}],"dateOfCollection":null,"lastUpdateTimeStamp":null,"indicators":{"citationImpact":{"citationCount":0.0,"influence":2.1921431E-9,"popularity":2.2912217E-9,"impulse":0.0,"citationClass":"C5","influenceClass":"C5","impulseClass":"C5","popularityClass":"C5"}},"projects":[{"id":"corda_____he::1c53907b78bb7a7765ce158190572b73","code":"101120393","acronym":"Sec4AI4Sec","title":"Cybersecurity for AI-Augmented Systems","funder":"European Commission","pids":[{"scheme":"doi","value":"10.3030/101120393"}]}],"organizations":[{"legalName":"Universita della Svizzera Italiana","acronym":"USI","id":"openorgs____::ef025332c8ad7e393d2a6932f765d0d3","pids":[{"scheme":"Wikidata","value":"Q689617"},{"scheme":"GRID","value":"grid.29078.34"},{"scheme":"PIC","value":"999585874"},{"scheme":"ISNI","value":"0000000122032861"},{"scheme":"wikidata","value":"Q27484979"},{"scheme":"Wikidata","value":"Q27484979"},{"scheme":"OrgRef","value":"2290642"},{"scheme":"ROR","value":"https://ror.org/03c4atk17"},{"scheme":"OrgReg","value":"CH0010"},{"scheme":"wikidata","value":"Q689617"},{"scheme":"mag_id","value":"57201433"}]},{"legalName":"SAP AG","acronym":"SAP AG","id":"pending_org_::0253607ffa35a07f0cfac48b80b2e8c7","pids":[{"scheme":"PIC","value":"999911212"}]}],"communities":null,"collectedFrom":[{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},{"key":"opendoar____::a1dc51ee503c5fe5170288a0b5bfc089","value":"SONAR - Swiss Open Access Repository"},{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"},{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}],"instances":[{"pids":[{"scheme":"doi","value":"10.1007/s10664-025-10743-w"}],"license":"CC BY","accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":"hybrid"},"type":"Article","urls":["https://doi.org/10.1007/s10664-025-10743-w"],"publicationDate":"2025-12-05","refereed":"peerReviewed","hostedBy":{"key":"issn___print::3902ffa9301f7ecc68932fec0e4339db","value":"Empirical Software Engineering"},"collectedFrom":{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}},{"pids":[{"scheme":"doi","value":"10.48550/arxiv.2411.18216"}],"license":"CC BY","type":"Article","urls":["https://dx.doi.org/10.48550/arxiv.2411.18216"],"publicationDate":"2024-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}},{"pids":[{"scheme":"arXiv","value":"2411.18216"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Preprint","urls":["http://arxiv.org/abs/2411.18216"],"publicationDate":"2024-11-27","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},"collectedFrom":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1007/s10664-025-10743-w"}],"type":"Article","urls":["https://doi.org/10.1007/s10664-025-10743-w"],"publicationDate":"2026-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2411.18216"}],"type":"Preprint","urls":["https://doi.org/10.48550/arXiv.2411.18216"],"publicationDate":"2024-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1007/s10664-025-10743-w"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","urls":["https://sonar.ch/global/documents/333995","https://n2t.net/ark:/12658/srd1333995","https://sonar.ch/documents/333995/files/s10664-025-10743-w.pdf"],"publicationDate":"2025-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::a1dc51ee503c5fe5170288a0b5bfc089","value":"SONAR - Swiss Open Access Repository"},"collectedFrom":{"key":"opendoar____::a1dc51ee503c5fe5170288a0b5bfc089","value":"SONAR - Swiss Open Access Repository"}}],"links":[{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"corda_____he::1c53907b78bb7a7765ce158190572b73","relatedRecordType":"project","relationProvenance":"iis","trust":"0.897"},"collectedfrom":[{"dsId":"openaire____::3f264f93cf3b0cfc4ede188a6300455c","dsName":"CORDA - COmmon Research DAta Warehouse - Horizon Europe"}],"projectTitle":"Cybersecurity for AI-Augmented Systems","code":"101120393","funding":{"funder":{"id":"ec__________::EC","shortname":"EC","name":"European Commission","jurisdiction":{"code":"EU","label":"European Union"},"pid":null},"level0":{"id":"ec__________::EC::HE","description":"Horizon Europe Framework Programme","name":"HE"},"level1":{"id":"ec__________::EC::HE::HORIZON-RIA","description":"HORIZON  Research and Innovation Actions","name":"HORIZON-RIA"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2023-10-01","endDate":"2026-09-30"}],"otherTitles":null,"inDiamondJournal":false,"green":true,"isGreen":true,"isInDiamondJournal":false}