Figured it out by myself:
#Prepare the excel we want to write to
t=pd.date_range('2004-01-31', freq='M', periods=4)
first=pd.DataFrame({'A':[1,1,1,1],
'B':[2,2,2,2]}, index=t)
first.index=first.index.strftime('%Y-%m-%d')
writer=pd.ExcelWriter('test.xlsx')
first.to_excel(writer, sheet_name="Here")
first.to_excel(writer, sheet_name="Keep")
#read the existing sheets so that openpyxl won't create a new one later
book = load_workbook('test.xlsx')
writer = pandas.ExcelWriter('test.xlsx', engine="openpyxl")
writer.book = book
writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
#update without overwrites
update=pd.DataFrame({'A':[3,4],
'B':[4,5]}, index=(pd.date_range('2004-04-30',
periods=2,
freq='M').strftime('%Y-%m-%d')))
update.to_excel(writer, "Here", startrow=1, startcol=2)
writer.save()