{"authors":[{"id":"orcid_______::34f0ece89a4d64e8c9ff5b7641756831","fullName":"Malmir, Mohammadhossein","name":"Mohammadhossein","surname":"Malmir","rank":1,"pid":{"id":{"scheme":"orcid","value":"0000-0003-0610-7899"},"provenance":null}},{"id":"orcid_______::5c6182f483af5fadc7e7b13ec3853b24","fullName":"Josifovski, Josip","name":"Josip","surname":"Josifovski","rank":2,"pid":{"id":{"scheme":"orcid","value":"0000-0002-1031-7621"},"provenance":null}},{"id":"orcid_______::e09111b8306ffb54fa08eb486670fb09","fullName":"Klarmann, Noah","name":"Noah","surname":"Klarmann","rank":3,"pid":{"id":{"scheme":"orcid","value":"0009-0008-9157-9228"},"provenance":null}},{"id":"orcid_______::1883e295acd90beb11a1de8239c6d2fe","fullName":"Knoll, Alois","name":"Alois","surname":"Knoll","rank":4,"pid":{"id":{"scheme":"orcid","value":"0000-0003-4840-076x"},"provenance":null}},{"id":null,"fullName":"Malmir, Mohammadhossein","name":"Mohammadhossein","surname":"Malmir","rank":5,"pid":null}],"openAccessColor":"hybrid","publiclyFunded":false,"eoscIfGuidelines":null,"type":"publication","language":{"code":"und","label":"Undetermined"},"countries":null,"subjects":[{"subject":{"scheme":"keyword","value":"Machine Learning"},"provenance":null},{"subject":{"scheme":"keyword","value":"FOS: Computer and information sciences"},"provenance":null},{"subject":{"scheme":"FOS","value":"0209 industrial biotechnology"},"provenance":null},{"subject":{"scheme":"keyword","value":"FOS: Electrical engineering, electronic engineering, information engineering"},"provenance":null},{"subject":{"scheme":"FOS","value":"02 engineering and technology"},"provenance":null},{"subject":{"scheme":"keyword","value":"Robotics"},"provenance":null},{"subject":{"scheme":"keyword","value":"Systems and Control (eess.SY)"},"provenance":null},{"subject":{"scheme":"keyword","value":"Robotics (cs.RO)"},"provenance":null},{"subject":{"scheme":"keyword","value":"Systems and Control"},"provenance":null},{"subject":{"scheme":"keyword","value":"Machine Learning (cs.LG)"},"provenance":null}],"mainTitle":"DiAReL: Reinforcement Learning With Disturbance Awareness for Robust Sim2Real Policy Transfer in Robot Control","subTitle":null,"descriptions":["Delayed Markov decision processes (DMDPs) fulfill the Markov property by augmenting the state space of agents with a finite time window of recently committed actions. In reliance on these state augmentations, delay-resolved reinforcement learning algorithms train policies to learn optimal interactions with environments featuring observation or action delays. Although such methods can be directly trained on the real robots, due to sample inefficiency, limited resources, or safety constraints, a common approach is to transfer models trained in simulation to the physical robot. However, robotic simulations rely on approximated models of the physical systems, which hinders the sim2real transfer. In this work, we consider various uncertainties in modeling the robot or environment dynamics as unknown intrinsic disturbances applied to the system input. We introduce the disturbance-augmented Markov decision process (DAMDP) in delayed settings as a novel representation to incorporate disturbance estimation in training on-policy reinforcement learning algorithms. The proposed method is validated across several metrics on learning robotic reaching and pushing tasks and compared with disturbance-unaware baselines. The results show that the disturbance-augmented models can achieve higher stabilization and robustness in the control response, which in turn improves the prospects of successful sim2real transfer.","Accepted for publication in IEEE Transactions on Control Systems Technology (TCST)"],"publicationDate":"2026-03-01","publisher":"Institute of Electrical and Electronics Engineers (IEEE)","embargoEndDate":"2023-06-01","sources":["Crossref","IEEE Transactions on Control Systems Technology"],"formats":null,"contributors":null,"coverages":null,"bestAccessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/"},"container":{"name":"IEEE Transactions on Control Systems Technology","issnPrinted":"1063-6536","issnOnline":"2374-0159","issnLinking":null,"ep":"1043","iss":null,"sp":"1037","vol":"34","edition":null,"conferencePlace":null,"conferenceDate":null},"documentationUrls":null,"codeRepositoryUrl":null,"programmingLanguage":null,"contactPeople":null,"contactGroups":null,"tools":null,"size":null,"version":null,"geoLocations":null,"id":"doi_dedup___::0156115c83c69cb32a7ba3267ea8f7fc","originalIds":["10.1109/tcst.2025.3634677","50|doiboost____|31a50519dc01c5104387ffa2152aeef4","50|datacite____::0156115c83c69cb32a7ba3267ea8f7fc","10.48550/arxiv.2306.09010","50|od________18::3dccf43218a89e634caf9b7fdc1516a9","oai:arXiv.org:2306.09010","50|dblp________::e30299466317c84527de8681db85d4bf","50|dblp________::e6f6c1c71c4aea98a2a1c51b45265630","50|r3c4b2081b22::7f351e1dd589c535efb85fef8542b392","10.48550/ARXIV.2306.09010","50|sygma_______::31a50519dc01c5104387ffa2152aeef4"],"pids":[{"scheme":"doi","value":"10.1109/tcst.2025.3634677"},{"scheme":"doi","value":"10.48550/arxiv.2306.09010"},{"scheme":"arXiv","value":"2306.09010"}],"dateOfCollection":null,"lastUpdateTimeStamp":null,"indicators":{"citationImpact":{"citationCount":0.0,"influence":2.1921431E-9,"popularity":2.6569356E-9,"impulse":0.0,"citationClass":"C5","influenceClass":"C5","impulseClass":"C5","popularityClass":"C5"}},"projects":[{"id":"corda_____he::0b3c3a221806b7844f34e7dfc7107f6e","code":"101096658","acronym":"A-IQ Ready","title":"Artificial Intelligence using Quantum measured Information for realtime distributed systems at the edge","funder":"European Commission","pids":[{"scheme":"doi","value":"10.3030/101096658"}]}],"organizations":[{"legalName":"University of Applied Sciences Rosenheim","acronym":"University of Applied Sciences Rosenheim","id":"openorgs____::29ed95ed584b596bff24d772ff63e936","pids":[{"scheme":"wikidata","value":"Q263537"},{"scheme":"GRID","value":"grid.449770.9"},{"scheme":"ROR","value":"https://ror.org/03hbmgt12"},{"scheme":"Wikidata","value":"Q263537"},{"scheme":"OrgRef","value":"21444947"},{"scheme":"ISNI","value":"0000000100586011"}]},{"legalName":"Technical University of Munich (TUM)","acronym":"Technical University of Munich (TUM)","id":"pending_org_::9dd178942479ad26449a2128190b1c3c","pids":null},{"legalName":"Offenburg University of Applied Sciences","acronym":"Offenburg University of Applied Sciences","id":"openorgs____::143710baf38394155543081faafed6df","pids":[{"scheme":"Wikidata","value":"Q1622133"},{"scheme":"ISNI","value":"0000000122346983"},{"scheme":"wikidata","value":"Q1622133"},{"scheme":"mag_id","value":"913140155"},{"scheme":"ROR","value":"https://ror.org/03zh5eq96"},{"scheme":"PIC","value":"961360405"},{"scheme":"GRID","value":"grid.440974.a"},{"scheme":"OrgRef","value":"31880821"}]},{"legalName":"Technical University of Munich","acronym":"TUM","id":"openorgs____::d04182cd5539c7709d5fadcef0cc08df","pids":[{"scheme":"FundRef","value":"501100013810"},{"scheme":"Wikidata","value":"Q157808"},{"scheme":"ROR","value":"https://ror.org/02kkvpp62"},{"scheme":"fundref","value":"501100005713"},{"scheme":"fundref","value":"501100013810"},{"scheme":"ISNI","value":"0000000123222966"},{"scheme":"RRID","value":"RRID:SCR_011560"},{"scheme":"FundRef","value":"501100009375"},{"scheme":"fundref","value":"501100013497"},{"scheme":"GRID","value":"grid.6936.a"},{"scheme":"wikidata","value":"Q157808"},{"scheme":"FundRef","value":"501100013497"},{"scheme":"fundref","value":"501100009375"},{"scheme":"FundRef","value":"501100005713"},{"scheme":"OrgRef","value":"349679"}]},{"legalName":"TECHNISCHE HOCHSCHULE ROSENHEIM / TECHNICAL UNIVERSITY OF APPLIED SCIENCES","acronym":"TECHNISCHE HOCHSCHULE ROSENHEIM","id":"pending_org_::ff15cd2d227db924fd13e57cc71f8552","pids":[{"scheme":"PIC","value":"955606365"}]},{"legalName":"TECHNISCHE UNIVERSITAET MUENCHEN","acronym":"TUM","id":"pending_org_::45501e9c377e7f4c0cdd7da4faf71d1d","pids":[{"scheme":"ROR","value":"https://ror.org/02kkvpp62"},{"scheme":"RRID","value":"RRID:nlx_156792"},{"scheme":"PIC","value":"999977463"},{"scheme":"mag_id","value":"62916508"},{"scheme":"RRID","value":"RRID:SCR_011560"}]}],"communities":null,"collectedFrom":[{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},{"key":"openaire____::a8db6f6b2ce4fe72e8b2314a9a93e7d9","value":"Sygma"},{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"},{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},{"key":"re3data_____::c4b2081b224be6b3e79d0e5e5556f631","value":"European Union Open Data Portal"}],"instances":[{"pids":[{"scheme":"doi","value":"10.1109/tcst.2025.3634677"}],"license":"CC BY","accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":"hybrid"},"type":"Article","urls":["https://doi.org/10.1109/tcst.2025.3634677"],"publicationDate":"2026-03-01","refereed":"peerReviewed","hostedBy":{"key":"issn___print::4ef452bd91d44d5ac556f6c368516574","value":"IEEE Transactions on Control Systems Technology"},"collectedFrom":{"key":"openaire____::081b82f96300b6a6e3d282bad31cb6e2","value":"Crossref"}},{"pids":[{"scheme":"doi","value":"10.48550/arxiv.2306.09010"}],"license":"arXiv Non-Exclusive Distribution","type":"Article","urls":["https://dx.doi.org/10.48550/arxiv.2306.09010"],"publicationDate":"2023-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::55045bd2a65019fd8e6741a755395c8c","value":"Unknown Repository"},"collectedFrom":{"key":"openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}},{"pids":[{"scheme":"arXiv","value":"2306.09010"}],"accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Preprint","urls":["http://arxiv.org/abs/2306.09010"],"publicationDate":"2023-06-15","refereed":"nonPeerReviewed","hostedBy":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"},"collectedFrom":{"key":"opendoar____::6f4922f45568161a8cdf4ad2299f6d23","value":"arXiv.org e-Print Archive"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2306.09010"}],"type":"Preprint","urls":["https://doi.org/10.48550/arXiv.2306.09010"],"publicationDate":"2023-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1109/tcst.2025.3634677"}],"type":"Article","urls":["https://doi.org/10.1109/TCST.2025.3634677"],"publicationDate":"2026-01-01","refereed":"nonPeerReviewed","hostedBy":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"},"collectedFrom":{"key":"openaire____::d8b68cc0a53121f6f896883a7d60c1db","value":"DBLP"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.48550/arxiv.2306.09010"}],"type":"Article","urls":["http://dx.doi.org/10.48550/ARXIV.2306.09010"],"publicationDate":"2025-01-01","refereed":"peerReviewed","hostedBy":{"key":"issn___print::4ef452bd91d44d5ac556f6c368516574","value":"IEEE Transactions on Control Systems Technology"},"collectedFrom":{"key":"re3data_____::c4b2081b224be6b3e79d0e5e5556f631","value":"European Union Open Data Portal"}},{"alternateIdentifiers":[{"scheme":"doi","value":"10.1109/tcst.2025.3634677"}],"license":"CC BY","accessRight":{"code":"c_abf2","label":"OPEN","scheme":"http://vocabularies.coar-repositories.org/documentation/access_rights/","openAccessRoute":null},"type":"Article","urls":["http://dx.doi.org/10.1109/tcst.2025.3634677"],"refereed":"nonPeerReviewed","hostedBy":{"key":"issn___print::4ef452bd91d44d5ac556f6c368516574","value":"IEEE Transactions on Control Systems Technology"},"collectedFrom":{"key":"openaire____::a8db6f6b2ce4fe72e8b2314a9a93e7d9","value":"Sygma"}}],"links":[{"header":{"relationType":"resultProject","relationClass":"isProducedBy","relatedIdentifier":"corda_____he::0b3c3a221806b7844f34e7dfc7107f6e","relatedRecordType":"project","relationProvenance":"sysimport:crosswalk","trust":"0.9"},"collectedfrom":[{"dsId":"openaire____::3f264f93cf3b0cfc4ede188a6300455c","dsName":"CORDA - COmmon Research DAta Warehouse - Horizon Europe"}],"projectTitle":"Artificial Intelligence using Quantum measured Information for realtime distributed systems at the edge","code":"101096658","funding":{"funder":{"id":"ec__________::EC","shortname":"EC","name":"European Commission","jurisdiction":{"code":"EU","label":"European Union"},"pid":null},"level0":{"id":"ec__________::EC::HE","description":"Horizon Europe Framework Programme","name":"HE"},"level1":{"id":"ec__________::EC::HE::HORIZON-JU-RIA","description":"HORIZON JU Research and Innovation Actions","name":"HORIZON-JU-RIA"},"level2":{"id":null,"description":null,"name":null}},"startDate":"2023-01-01","endDate":"2026-03-31"}],"otherTitles":null,"inDiamondJournal":false,"green":true,"isGreen":true,"isInDiamondJournal":false}