{"authors":[{"id":"orcid_______::d935e8e6d7d98847e8f42d1d49ab0afa","fullName":"Carreto Picón, Ginés","name":"Ginés","surname":"Carreto Picón","rank":1,"pid":{"id":{"scheme":"orcid","value":"0000-0001-7844-9711"},"provenance":null}},{"id":"orcid_______::c9d06372809fad6d05e01e6c90f0a04b","fullName":"Oleksiienko, Illia","name":"Illia","surname":"Oleksiienko","rank":2,"pid":{"id":{"scheme":"orcid","value":"0000-0001-7592-365x"},"provenance":null}},{"id":"orcid_______::dc755e900eb98c1fd2b524456b05800e","fullName":"Hedegaard, Lukas","name":"Lukas","surname":"Hedegaard","rank":3,"pid":{"id":{"scheme":"orcid","value":"0000-0002-2841-864x"},"provenance":null}},{"id":"orcid_______::bbdbe86c1bee25ac259774978dcbacc3","fullName":"Bakhtiarnia, Arian","name":"Arian","surname":"Bakhtiarnia","rank":4,"pid":{"id":{"scheme":"orcid","value":"0000-0001-8624-8661"},"provenance":null}},{"id":"orcid_______::c00e59a636684a5db67b69fef6da4c22","fullName":"Iosifidis, Alexandros","name":"Alexandros","surname":"Iosifidis","rank":5,"pid":{"id":{"scheme":"orcid","value":"0000-0003-4807-1345"},"provenance":null}}],"openAccessColor":"hybrid","publiclyFunded":false,"eoscIfGuidelines":null,"type":"publication","language":{"code":"eng","label":"English"},"countries":[{"code":"FI","label":"Finland","provenance":null}],"subjects":[{"subject":{"scheme":"keyword","value":"FOS: Computer and information sciences"},"provenance":null},{"subject":{"scheme":"keyword","value":"Transformer"},"provenance":null},{"subject":{"scheme":"keyword","value":"Nyström approximation"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Vision and Pattern Recognition (cs.CV)"},"provenance":null},{"subject":{"scheme":"keyword","value":"continual inference"},"provenance":null},{"subject":{"scheme":"keyword","value":"nyström approximation"},"provenance":null},{"subject":{"scheme":"keyword","value":"deep learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"Deep learning"},"provenance":null},{"subject":{"scheme":"FOS","value":"02 engineering and technology"},"provenance":null},{"subject":{"scheme":"keyword","value":"Continual inference"},"provenance":null},{"subject":{"scheme":"keyword","value":"113"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (cs.LG)"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning"},"provenance":null},{"subject":{"scheme":"FOS","value":"0202 electrical engineering, electronic engineering, information engineering"},"provenance":null},{"subject":{"scheme":"keyword","value":"transformer"},"provenance":null},{"subject":{"scheme":"keyword","value":"Scaled dot-product attention"},"provenance":null},{"subject":{"scheme":"keyword","value":"Computer Vision and Pattern Recognition"},"provenance":null},{"subject":{"scheme":"keyword","value":"scaled dot-product attention"},"provenance":null}],"mainTitle":"Continual low-rank scaled dot-product attention","subTitle":null,"descriptions":["Transformers are widely used for their ability to capture data relations in sequence processing, with great success for a wide range of static tasks. However, the computational and memory footprint of their main component, i.e., the Scaled Dot-product Attention, is commonly overlooked. This makes their adoption in applications involving stream data processing with constraints in response latency, computational and memory resources infeasible. Some works have proposed methods to lower the computational cost of Transformers, i.e. low-rank approximations, sparsity in attention, and efficient formulations for Continual Inference. In this paper, we introduce a new formulation of the Scaled Dot-product Attention based on the Nyström approximation that is suitable for Continual Inference. In experiments on Online Audio Classification and Online Action Detection tasks, the proposed Continual Scaled Dot-product Attention can lower the number of operations by up to three orders of magnitude compared to the original Transformers while retaining the predictive performance of competing models.","16 pages, 7 figures"],"publicationDate":"2026-05-01","publisher":"Elsevier BV","embargoEndDate":"2024-12-01","sources":["Crossref","Carreto Picón, G, Oleksiienko, I, Hedegaard, L, Bakhtiarnia, A & Iosifidis, A 2026, 'Continual low-rank scaled dot-product attention', Neural Networks, vol. 197, 108517. https://doi.org/10.1016/j.neunet.2025.108517","Carreto Picón, G, Oleksiienko, I, Morsing, L H, Bakhtiarnia, A & Iosifidis, A 2024 'Continual Low-Rank Scaled Dot-product Attention'. https://doi.org/10.48550/arXiv.2412.03214"],"formats":["fulltext"],"contributors":["Tampere University","Tietotekniikka","Computing Sciences"],"coverages":null,"bestAccessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"Neural Networks","issnPrinted":"0893-6080","issnOnline":null,"issnLinking":null,"ep":null,"iss":null,"sp":"108517","vol":"197","edition":null,"conferencePlace":null,"conferenceDate":null},"documentationUrls":null,"codeRepositoryUrl":null,"programmingLanguage":null,"contactPeople":null,"contactGroups":null,"tools":null,"size":null,"version":null,"geoLocations":null,"id":"doi_dedup___::e30e7df3c2244a3693aeb4aaa56f496b","originalIds":["S089360802501398X","10.1016/j.neunet.2025.108517","50|doiboost____|a9c872fe20ba5e73a901f4eed5f3fb59","50|datacite____::e30e7df3c2244a3693aeb4aaa56f496b","10.48550/arxiv.2412.03214","41475131","oai:arXiv.org:2412.03214","50|od________18::354b9d63372d639d7b1bc0c1341877bc","50|dblp________::8d0f9c0db0648f70a0065261ed0b3247","50|dblp________::e4360fb503230963a2ad3f293fdbf20d","oai:trepo.tuni.fi:10024/234000","50|od______4853::dd461154bb9275f77f89cd793d33159b","oai:pure.atira.dk:publications/b3847b7a-6c63-468b-9824-230f9a9eb945","50|pure_au_____::32cdafa2e0bac68469d28b3271e7aaac","oai:pure.atira.dk:publications/20d511dd-c03f-400b-9390-28ea55b0523b","50|pure_au_____::e1be98c35e4716f589fa70aa3f3df882","50|researchfi__::62a26c433ea7197cf3a398592db00653","50|sygma_______::a9c872fe20ba5e73a901f4eed5f3fb59"],"pids":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"},{"scheme":"doi","value":"10.48550/arxiv.2412.03214"},{"scheme":"pmid","value":"41475131"},{"scheme":"arXiv","value":"2412.03214"}],"dateOfCollection":null,"lastUpdateTimeStamp":null,"indicators":{"citationImpact":{"citationCount":0.0,"influence":2.2251732E-9,"popularity":2.7165745E-9,"impulse":0.0,"citationClass":"C5","influenceClass":"C5","impulseClass":"C5","popularityClass":"C5"}},"projects":[{"id":"corda_____he::679ba2587b03b4e6c4ca150b628d6975","code":"101135775","acronym":"PANDORA","title":"A Comprehensive Framework enabling the Delivery of Trustworthy Datasets for Efficient AIoT Operation","funder":"European Commission","pids":[{"scheme":"doi","value":"10.3030/101135775"}]}],"organizations":[{"legalName":"Tampere University","acronym":"Tampere University","id":"openorgs____::5e6abea3de669950a536a47ae6f134c1","pids":[{"scheme":"ROR","value":"https://ror.org/05p6qqk79"},{"scheme":"PIC","value":"999903161"},{"scheme":"OrgReg","value":"FI2001"},{"scheme":"FundRef","value":"501100004371"},{"scheme":"GRID","value":"grid.502801.e"},{"scheme":"PIC","value":"902999288"},{"scheme":"OrgReg","value":"FI0005"},{"scheme":"ISNI","value":"0000000123146254"},{"scheme":"RRID","value":"RRID:SCR_011709"},{"scheme":"ISNI","value":"0000000507186722"},{"scheme":"fundref","value":"501100004371"},{"scheme":"ROR","value":"https://ror.org/033003e23"},{"scheme":"GRID","value":"grid.5509.9"},{"scheme":"mag_id","value":"166825849"},{"scheme":"wikidata","value":"Q56403399"},{"scheme":"wikidata","value":"Q1544881"},{"scheme":"OrgRef","value":"606022"},{"scheme":"Wikidata","value":"Q56403399"},{"scheme":"Wikidata","value":"Q1544881"}]},{"legalName":"Aarhus University","acronym":"Aarhus University","id":"pending_org_::1e84740704965e787fc4a5a4443fc154","pids":null},{"legalName":"Aarhus University","acronym":"Aarhus University","id":"pending_org_::e6cfb9bcacf1902bb68fc2d244627e85","pids":null},{"legalName":"Aarhus University","acronym":"Aarhus University","id":"pending_org_::75e0d8fbc6c291f0e5ac7ff1a264cd7d","pids":null},{"legalName":"Aarhus University","acronym":"AU","id":"openorgs____::d4ceca70c60317a8c1e79c4dd0260a31","pids":[{"scheme":"fundref","value":"501100006308"},{"scheme":"RRID","value":"RRID:SCR_000948"},{"scheme":"Wikidata","value":"Q924265"},{"scheme":"fundref","value":"501100006420"},{"scheme":"mag_id","value":"204337017"},{"scheme":"GRID","value":"grid.7048.b"},{"scheme":"wikidata","value":"Q924265"},{"scheme":"CVR","value":"41826614"},{"scheme":"OrgRef","value":"401280"},{"scheme":"FundRef","value":"501100006420"},{"scheme":"OrgReg","value":"DK0002"},{"scheme":"fundref","value":"501100007205"},{"scheme":"fundref","value":"100007605"},{"scheme":"fundref","value":"100008628"},{"scheme":"CVR","value":"31119103"},{"scheme":"FundRef","value":"501100006308"},{"scheme":"OrgRef","value":"38530302"},{"scheme":"PIC","value":"996342000"},{"scheme":"FundRef","value":"100013225"},{"scheme":"PIC","value":"999997736"},{"scheme":"ROR","value":"https://ror.org/01aj84f44"},{"scheme":"FundRef","value":"501100007205"},{"scheme":"RRID","value":"RRID:nlx_156945"},{"scheme":"FundRef","value":"100015423"},{"scheme":"fundref","value":"100015423"},{"scheme":"FundRef","value":"100012526"},{"scheme":"fundref","value":"501100002739"},{"scheme":"ISNI","value":"0000000119562722"},{"scheme":"fundref","value":"100012526"},{"scheme":"FundRef","value":"501100002739"},{"scheme":"FundRef","value":"100007605"},{"scheme":"fundref","value":"100013225"},{"scheme":"FundRef","value":"100008628"}]}],"communities":[{"code":"uarctic","label":"UArctic","provenance":null}],"collectedFrom":[{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},{"key":"openaire____::ddcdf7dcef98c2a7626bbb11d1a7b534","value":"Research.fi"},{"key":"openaire____::a8db6f6b2ce4fe72e8b2314a9a93e7d9","value":"Sygma"},{"key":"openaire____::d76e4d42b3bd658259e8bf9c37ef448f","value":"PURE Aarhus University"},{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"},{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},{"key":"opendoar____::8b6dd7db9af49e67306feb59a8bdc52c","value":"Europe PubMed Central"},{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},{"key":"opendoar____::812214fb8e7066bfa6e32c626c2c688b","value":"Trepo - Institutional Repository of Tampere University"}],"instances":[{"pids":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"}],"license":"CC BY","accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":"hybrid"},"type":"Article","urls":["https://doi.org/10.1016/j.neunet.2025.108517"],"publicationDate":"2026-05-01","refereed":"peerReviewed","hostedBy":{"key":"issn___print::0e0caf76228e4f49588922d126eff1fe","value":"Neural Networks"},"collectedFrom":{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}},{"pids":[{"scheme":"doi","value":"10.48550/arxiv.2412.03214"}],"license":"arXiv Non-Exclusive Distribution","type":"Article","urls":["https://dx.doi.org/10.48550/arxiv.2412.03214"],"publicationDate":"2024-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}},{"pids":[{"scheme":"pmid","value":"41475131"}],"alternateIdentifiers":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"}],"type":"Article","urls":["https://pubmed.ncbi.nlm.nih.gov/41475131"],"refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"opendoar____::8b6dd7db9af49e67306feb59a8bdc52c","value":"Europe PubMed Central"}},{"pids":[{"scheme":"arXiv","value":"2412.03214"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Preprint","urls":["http://arxiv.org/abs/2412.03214"],"publicationDate":"2024-12-04","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},"collectedFrom":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2412.03214"}],"type":"Preprint","urls":["https://doi.org/10.48550/arXiv.2412.03214"],"publicationDate":"2024-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"}],"type":"Article","urls":["https://doi.org/10.1016/j.neunet.2025.108517"],"publicationDate":"2026-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"}],"type":"Article","urls":["https://trepo.tuni.fi/handle/10024/234000"],"publicationDate":"2026-01-27","refereed":"peerReviewed","hostedBy":{"key":"opendoar____::812214fb8e7066bfa6e32c626c2c688b","value":"Trepo - Institutional Repository of Tampere University"},"collectedFrom":{"key":"opendoar____::812214fb8e7066bfa6e32c626c2c688b","value":"Trepo - Institutional Repository of Tampere University"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"},{"scheme":"pmid","value":"41475131"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","urls":["https://pure.au.dk/portal/en/publications/b3847b7a-6c63-468b-9824-230f9a9eb945","https://www.scopus.com/pages/publications/105026143948","https://doi.org/10.1016/j.neunet.2025.108517"],"publicationDate":"2026-05-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d76e4d42b3bd658259e8bf9c37ef448f","value":"PURE Aarhus University"},"collectedFrom":{"key":"openaire____::d76e4d42b3bd658259e8bf9c37ef448f","value":"PURE Aarhus University"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2412.03214"}],"accessRight":{"code":"c_16ec","label":"RESTRICTED","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Research","urls":["https://doi.org/10.48550/arXiv.2412.03214","https://pure.au.dk/portal/en/publications/20d511dd-c03f-400b-9390-28ea55b0523b"],"publicationDate":"2024-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d76e4d42b3bd658259e8bf9c37ef448f","value":"PURE Aarhus University"},"collectedFrom":{"key":"openaire____::d76e4d42b3bd658259e8bf9c37ef448f","value":"PURE Aarhus University"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"}],"license":"CC BY","accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","urls":["https://doi.org/10.1016/j.neunet.2025.108517"],"publicationDate":"2026-01-29","refereed":"peerReviewed","hostedBy":{"key":"openaire____::ddcdf7dcef98c2a7626bbb11d1a7b534","value":"Research.fi"},"collectedFrom":{"key":"openaire____::ddcdf7dcef98c2a7626bbb11d1a7b534","value":"Research.fi"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1016/j.neunet.2025.108517"}],"license":"Elsevier TDM","accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","urls":["http://dx.doi.org/10.1016/j.neunet.2025.108517"],"refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::a8db6f6b2ce4fe72e8b2314a9a93e7d9","value":"Sygma"}}],"links":[{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"corda_____he::679ba2587b03b4e6c4ca150b628d6975","relatedRecordType":"project","relationProvenance":"iis","trust":"0.897"},"collectedfrom":[{"dsId":"openaire____::3f264f93cf3b0cfc4ede188a6300455c","dsName":"CORDA - COmmon Research DAta Warehouse - Horizon Europe"}],"projectTitle":"A Comprehensive Framework enabling the Delivery of Trustworthy Datasets for Efficient AIoT Operation","code":"101135775","funding":{"funder":{"id":"ec__________::EC","shortname":"EC","name":"European Commission","jurisdiction":{"code":"EU","label":"European Union"},"pid":null},"level0":{"id":"ec__________::EC::HE","description":"Horizon Europe Framework Programme","name":"HE"},"level1":{"id":"ec__________::EC::HE::HORIZON-RIA","description":"HORIZON  Research and Innovation Actions","name":"HORIZON-RIA"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2024-04-01","endDate":"2027-03-31"}],"otherTitles":null,"inDiamondJournal":false,"green":true,"isGreen":true,"isInDiamondJournal":false}