http://www.chakkaradeep.com/post/Using-the-SharePoint-2010-Modal-Dialog.aspx is a good example.
Basically you create the Modal dialog like so...
//Dialog Opening
function OpenDialog() {
var options = SP.UI.$create_DialogOptions();
options.url = '/_layouts/mySolution/myPage.aspx';
options.width = 500;
options.height = 400;
options.dialogReturnValueCallback = Function.createDelegate(null, CloseCallback);
SP.UI.ModalDialog.showModalDialog(options);
}
and then make the return function like this...
// Dialog callback
function CloseCallback(result, target) {
if (result == SP.UI.DialogResult.OK) {
// Run OK Code
}
if (result == SP.UI.DialogResult.cancel) {
// Run Cancel Code
}
}