We had this problem on a custom sharepoint site and fixed it by adding a min-width.
This is not the best solution but it works:
.ms-dialog BODY #s4-bodyContainer
{
width: auto !important;
min-width: 660px !important;
padding: 3px;
}
.ms-dialog BODY .no-menu#MSO_ContentTable { padding: 0px; }
When I have some time, I will dig deep into how sharepoint tries to calculate this width, and hopefully find a better fix.
If you wish to investigate it yourself look on line 554 in sp.ui.dialog.debug.js, you will find this code, and it will be really hard to read, but give it a go.
autoSize: function() {ULSTYE:;
var $v_0 = 0;
var $v_1 = 0;
this.$Q_0((SP.ScriptUtility.isNullOrUndefined(this.$G_0.width)) ? 500 : this.$G_0.width, (SP.ScriptUtility.isNullOrUndefined(this.$G_0.height)) ? 20 : this.$G_0.height);
if (this.$0_0) {
try {
var $v_2 = this.$0_0.contentWindow.document;
var $v_3 = $v_2.getElementById('s4-ribbonrow');
var $v_4 = $v_2.getElementById('s4-workspace');
if ($v_3 && $v_4) {
var $v_5 = $v_4.style;
var $v_6 = $v_3.style;
var $v_7 = $v_5.width;
var $v_8 = $v_5.height;
var $v_9 = $v_6.width;
var $v_A = $v_6.height;
$v_5.width = '';
$v_5.height = '';
$v_6.width = '';
$v_6.height = '';
$v_0 = Math.max($v_3.offsetWidth, $v_4.scrollWidth);
$v_1 = $v_3.offsetHeight + $v_4.scrollHeight;
$v_5.width = $v_7;
$v_5.height = $v_8;
$v_6.width = $v_9;
$v_6.height = $v_A;
}
else {
var $v_B = $v_2.getElementById('ms-notdlgautosize');
if ($v_B) {
$v_0 = parseInt(SP.Res.defaultDialogWidth);
$v_1 = parseInt(SP.Res.defaultDialogHeight);
}
else {
$v_0 = this.$0_0.contentWindow.document.body.scrollWidth;
$v_1 = this.$0_0.contentWindow.document.body.scrollHeight;
}
}
}
catch ($$e_2_0) {
}
}
else if (this.$E_0) {
$v_0 = this.$E_0.scrollWidth;
$v_1 = this.$E_0.scrollHeight;
}
$v_0 += SP.UI.Dialog.$1b_0();
$v_1 += SP.UI.Dialog.$1b_0();
if ($v_0 < 50) {
$v_0 = parseInt(SP.Res.defaultDialogWidth);
}
if ($v_1 < 20) {
$v_1 = parseInt(SP.Res.defaultDialogHeight);
}
if (SP.ScriptUtility.isNullOrUndefined(this.$G_0.height)) {
this.$A_0 = $v_1;
}
if (SP.ScriptUtility.isNullOrUndefined(this.$G_0.width)) {
this.$B_0 = $v_0;
}
this.$Q_0(this.$B_0, this.$A_0);
}